Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj draft
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 31 May 2018 16:36:05 +0900
changeset 806987 905ab9e02bafeafbf5d996912a2e16409f53f97e
parent 806951 49efc43b14387db6c68d70bbf5e9b8195dd7f80f
push id113013
push userbmo:mh+mozilla@glandium.org
push dateWed, 13 Jun 2018 04:36:05 +0000
reviewersfroydnj
bugs1465709, 1458161, 50880, 51264, 51241
milestone62.0a1
Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj Bug 1458161 added a rust OOM handler based on an unstable API that was removed in 1.27, replaced with something that didn't allow to get the failed allocation size. Latest 1.28 nightly (2018-06-13) has https://github.com/rust-lang/rust/pull/50880, https://github.com/rust-lang/rust/pull/51264 and https://github.com/rust-lang/rust/pull/51241 merged, which allow to hook the OOM handler and get the failed allocation size again. Because this is still an unstable API, we explicitly depend on strict versions of rustc. We also explicitly error out if automation builds end up using a rustc version that doesn't allow us to get the allocation size for rust OOM, because we don't want that to happen without knowing.
toolkit/crashreporter/nsExceptionHandler.cpp
toolkit/library/gtest/rust/Cargo.toml
toolkit/library/rust/Cargo.toml
toolkit/library/rust/gkrust-features.mozbuild
toolkit/library/rust/shared/Cargo.toml
toolkit/library/rust/shared/build.rs
toolkit/library/rust/shared/lib.rs
--- a/toolkit/crashreporter/nsExceptionHandler.cpp
+++ b/toolkit/crashreporter/nsExceptionHandler.cpp
@@ -112,16 +112,17 @@ using google_breakpad::FileID;
 using google_breakpad::PageAllocator;
 #endif
 using namespace mozilla;
 using mozilla::ipc::CrashReporterClient;
 
 // From toolkit/library/rust/shared/lib.rs
 extern "C" {
   void install_rust_panic_hook();
+  void install_rust_oom_hook();
   bool get_rust_panic_reason(char** reason, size_t* length);
 }
 
 
 namespace CrashReporter {
 
 #ifdef XP_WIN32
 typedef wchar_t XP_CHAR;
@@ -1682,16 +1683,18 @@ nsresult SetExceptionHandler(nsIFile* aX
 #endif
 
   mozalloc_set_oom_abort_handler(AnnotateOOMAllocationSize);
 
   oldTerminateHandler = std::set_terminate(&TerminateHandler);
 
   install_rust_panic_hook();
 
+  install_rust_oom_hook();
+
   InitThreadAnnotation();
 
   return NS_OK;
 }
 
 bool GetEnabled()
 {
   return gExceptionHandler != nullptr;
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -9,16 +9,17 @@ description = "Testing code for libgkrus
 bindgen = ["gkrust-shared/bindgen"]
 servo = ["gkrust-shared/servo"]
 quantum_render = ["gkrust-shared/quantum_render"]
 cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
 cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]
 mp4parse-gtest = { path = "../../../../dom/media/gtest" }
 nsstring-gtest = { path = "../../../../xpcom/rust/gtest/nsstring" }
 xpcom-gtest = { path = "../../../../xpcom/rust/gtest/xpcom" }
 gkrust-shared = { path = "../../rust/shared" }
 
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -9,16 +9,17 @@ description = "Rust code for libxul"
 bindgen = ["gkrust-shared/bindgen"]
 servo = ["gkrust-shared/servo"]
 quantum_render = ["gkrust-shared/quantum_render"]
 cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
 cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]
 gkrust-shared = { path = "shared" }
 
 [dev-dependencies]
 stylo_tests = { path = "../../../servo/ports/geckolib/tests/" }
 
--- a/toolkit/library/rust/gkrust-features.mozbuild
+++ b/toolkit/library/rust/gkrust-features.mozbuild
@@ -25,8 +25,15 @@ if (CONFIG['OS_ARCH'] == 'Linux' and CON
 
 if CONFIG['MOZ_MEMORY']:
     gkrust_features += ['moz_memory']
 
 # See details in toolkit/library/rust/shared/lib.rs
 # A string test is not the best thing, but it works well enough here.
 if CONFIG['RUSTC_VERSION'] < "1.27":
     gkrust_features += ['oom_with_global_alloc']
+elif CONFIG['RUSTC_VERSION'] >= "1.28" and CONFIG['RUSTC_VERSION'] < "1.29":
+    gkrust_features += ['oom_with_hook']
+elif not CONFIG['MOZ_AUTOMATION']:
+    # We don't want builds on automation to unwillingly stop annotating OOM
+    # crash reports from rust.
+    error('Builds on automation must use a version of rust that supports OOM '
+          'hooking')
--- a/toolkit/library/rust/shared/Cargo.toml
+++ b/toolkit/library/rust/shared/Cargo.toml
@@ -33,16 +33,17 @@ default = []
 bindgen = ["geckoservo/bindgen"]
 servo = ["geckoservo"]
 quantum_render = ["webrender_bindings"]
 cubeb-remoting = ["cubeb-sys", "audioipc-client", "audioipc-server"]
 cubeb_pulse_rust = ["cubeb-sys", "cubeb-pulse"]
 gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 oom_with_global_alloc = []
+oom_with_hook = []
 moz_memory = ["mp4parse_capi/mp4parse_fallible"]
 
 [lib]
 path = "lib.rs"
 test = false
 doctest = false
 bench = false
 doc = false
--- a/toolkit/library/rust/shared/build.rs
+++ b/toolkit/library/rust/shared/build.rs
@@ -1,8 +1,8 @@
 fn main() {
     // This is a rather awful thing to do, but we're only doing it on
     // versions of rustc, >= 1.24 < 1.27, that are not going to change
     // the unstable APIs we use from under us (1.26 being a beta as of
     // writing, and close to release).
-    #[cfg(feature = "oom_with_global_alloc")]
+    #[cfg(any(feature = "oom_with_global_alloc", feature = "oom_with_hook"))]
     println!("cargo:rustc-env=RUSTC_BOOTSTRAP=1");
 }
--- a/toolkit/library/rust/shared/lib.rs
+++ b/toolkit/library/rust/shared/lib.rs
@@ -1,14 +1,15 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 #![cfg_attr(feature = "oom_with_global_alloc",
             feature(global_allocator, alloc, alloc_system, allocator_api))]
+#![cfg_attr(feature = "oom_with_hook", feature(oom_hook))]
 
 #[cfg(feature="servo")]
 extern crate geckoservo;
 
 extern crate mp4parse_capi;
 extern crate nsstring;
 extern crate nserror;
 extern crate xpcom;
@@ -213,8 +214,33 @@ mod global_alloc {
         }
     }
 
 }
 
 #[cfg(feature = "oom_with_global_alloc")]
 #[global_allocator]
 static HEAP: global_alloc::GeckoHeap = global_alloc::GeckoHeap;
+
+#[cfg(feature = "oom_with_hook")]
+mod oom_hook {
+    use std::alloc::{Layout, set_oom_hook};
+
+    extern "C" {
+        fn GeckoHandleOOM(size: usize) -> !;
+    }
+
+    pub fn hook(layout: Layout) {
+        unsafe {
+            GeckoHandleOOM(layout.size());
+        }
+    }
+
+    pub fn install() {
+        set_oom_hook(hook);
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn install_rust_oom_hook() {
+    #[cfg(feature = "oom_with_hook")]
+    oom_hook::install();
+}