Bug 1458161 - Hook rust OOM handler. r=froydnj draft
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 01 May 2018 10:30:03 +0900
changeset 790385 63ed3ea65cef84c444d628bc923396f05799076d
parent 790006 7ef8450810693ab08e79ab0d4702de6f479e678c
push id108507
push userbmo:mh+mozilla@glandium.org
push dateWed, 02 May 2018 07:54:54 +0000
reviewersfroydnj
bugs1458161
milestone61.0a1
Bug 1458161 - Hook rust OOM handler. r=froydnj OOM rust crashes are currently not identified as such in crash reports because rust libstd handles the OOMs and panics itself. There are unstable ways to hook into this, which unfortunately are under active changes in rust 1.27, but we're currently on 1.24 and 1.27 is not released yet. The APIs didn't change between 1.24 and 1.26, so it's fine-ish to use them as long as we limit their use to those versions. As long as the Firefox versions we ship (as opposed to downstream) use the "right" version of rust, we're good to go. The APIs are in their phase of stabilization, so there shouldn't be too many variants of the code to support.
build/moz.configure/rust.configure
toolkit/library/gtest/rust/Cargo.toml
toolkit/library/rust/Cargo.toml
toolkit/library/rust/gkrust-features.mozbuild
toolkit/library/rust/shared/Cargo.toml
toolkit/library/rust/shared/build.rs
toolkit/library/rust/shared/lib.rs
toolkit/xre/nsAppRunner.cpp
--- a/build/moz.configure/rust.configure
+++ b/build/moz.configure/rust.configure
@@ -249,16 +249,18 @@ def rust_target_env_name(triple):
 # We need this to form various Cargo environment variables, as there is no
 # uppercase function in make, and we don't want to shell out just for
 # converting a string to uppercase.
 set_config('RUST_TARGET_ENV_NAME', rust_target_env_name)
 
 # This is used for putting source info into symbol files.
 set_config('RUSTC_COMMIT', depends(rustc_info)(lambda i: i.commit))
 
+set_config('RUSTC_VERSION', depends(rustc_info)(lambda i: str(i.version)))
+
 # Until we remove all the other Rust checks in old-configure.
 add_old_configure_assignment('RUSTC', rustc)
 add_old_configure_assignment('RUST_TARGET', rust_target_triple)
 
 # This option is separate from --enable-tests because Rust tests are particularly
 # expensive in terms of compile time (especially for code in libxul).
 option('--enable-rust-tests',
        help='Enable building of Rust tests, and build-time execution of them')
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -9,16 +9,17 @@ description = "Testing code for libgkrus
 bindgen = ["gkrust-shared/bindgen"]
 servo = ["gkrust-shared/servo"]
 quantum_render = ["gkrust-shared/quantum_render"]
 cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
 cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
 
 [dependencies]
 mp4parse-gtest = { path = "../../../../dom/media/gtest" }
 nsstring-gtest = { path = "../../../../xpcom/rust/gtest/nsstring" }
 xpcom-gtest = { path = "../../../../xpcom/rust/gtest/xpcom" }
 gkrust-shared = { path = "../../rust/shared" }
 
 [lib]
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -9,16 +9,17 @@ description = "Rust code for libxul"
 bindgen = ["gkrust-shared/bindgen"]
 servo = ["gkrust-shared/servo"]
 quantum_render = ["gkrust-shared/quantum_render"]
 cubeb-remoting = ["gkrust-shared/cubeb-remoting"]
 cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 no-static-ideograph-encoder-tables = ["gkrust-shared/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
 
 [dependencies]
 gkrust-shared = { path = "shared" }
 
 [dev-dependencies]
 stylo_tests = { path = "../../../servo/ports/geckolib/tests/" }
 
 [lib]
--- a/toolkit/library/rust/gkrust-features.mozbuild
+++ b/toolkit/library/rust/gkrust-features.mozbuild
@@ -19,8 +19,13 @@ if CONFIG['MOZ_RUST_SIMD']:
     gkrust_features += ['simd-accel']
 
 # This feature is only supported on Linux and macOS, and this check needs to
 # match MOZ_CUBEB_REMOTING in CubebUtils.cpp.
 if (CONFIG['OS_ARCH'] == 'Linux' and CONFIG['OS_TARGET'] != 'Android') or CONFIG['OS_ARCH'] == 'Darwin':
     gkrust_features += ['cubeb-remoting']
 
 gkrust_features += ['no-static-ideograph-encoder-tables']
+
+# See details in toolkit/library/rust/shared/lib.rs
+# A string test is not the best thing, but it works well enough here.
+if CONFIG['RUSTC_VERSION'] < "1.27":
+    gkrust_features += ['oom_with_global_alloc']
--- a/toolkit/library/rust/shared/Cargo.toml
+++ b/toolkit/library/rust/shared/Cargo.toml
@@ -33,16 +33,17 @@ default = []
 bindgen = ["geckoservo/bindgen"]
 servo = ["geckoservo"]
 quantum_render = ["webrender_bindings"]
 cubeb-remoting = ["cubeb-sys", "audioipc-client", "audioipc-server"]
 cubeb_pulse_rust = ["cubeb-sys", "cubeb-pulse"]
 gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 no-static-ideograph-encoder-tables = ["encoding_c/no-static-ideograph-encoder-tables", "encoding_glue/no-static-ideograph-encoder-tables"]
+oom_with_global_alloc = []
 
 [lib]
 path = "lib.rs"
 test = false
 doctest = false
 bench = false
 doc = false
 plugin = false
new file mode 100644
--- /dev/null
+++ b/toolkit/library/rust/shared/build.rs
@@ -0,0 +1,8 @@
+fn main() {
+    // This is a rather awful thing to do, but we're only doing it on
+    // versions of rustc, >= 1.24 < 1.27, that are not going to change
+    // the unstable APIs we use from under us (1.26 being a beta as of
+    // writing, and close to release).
+    #[cfg(feature = "oom_with_global_alloc")]
+    println!("cargo:rustc-env=RUSTC_BOOTSTRAP=1");
+}
--- a/toolkit/library/rust/shared/lib.rs
+++ b/toolkit/library/rust/shared/lib.rs
@@ -1,12 +1,15 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+#![cfg_attr(feature = "oom_with_global_alloc",
+            feature(global_allocator, alloc, alloc_system, allocator_api))]
+
 #[cfg(feature="servo")]
 extern crate geckoservo;
 
 extern crate mp4parse_capi;
 extern crate nsstring;
 extern crate nserror;
 extern crate xpcom;
 extern crate netwerk_helper;
@@ -154,8 +157,64 @@ pub extern "C" fn get_rust_panic_reason(
             *reason = s as *const c_char;
             *length = (*s).len();
             true
         } else {
             false
         }
     }
 }
+
+// Wrap the rust system allocator to override the OOM handler, redirecting
+// to Gecko's, which interacts with the crash reporter.
+// This relies on unstable APIs that have not changed between 1.24 and 1.27.
+// In 1.27, the API changed, so we'll need to adapt to those changes before
+// we can ship with 1.27. As of writing, there might still be further changes
+// to those APIs before 1.27 is released, so we wait for those.
+#[cfg(feature = "oom_with_global_alloc")]
+mod global_alloc {
+    extern crate alloc;
+    extern crate alloc_system;
+
+    use self::alloc::allocator::{Alloc, AllocErr, Layout};
+    use self::alloc_system::System;
+
+    pub struct GeckoHeap;
+
+    extern "C" {
+        fn GeckoHandleOOM(size: usize) -> !;
+    }
+
+    unsafe impl<'a> Alloc for &'a GeckoHeap {
+        unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
+            System.alloc(layout)
+        }
+
+        unsafe fn dealloc(&mut self, ptr: *mut u8, layout: Layout) {
+            System.dealloc(ptr, layout)
+        }
+
+        fn oom(&mut self, e: AllocErr) -> ! {
+            match e {
+                AllocErr::Exhausted { request } => unsafe { GeckoHandleOOM(request.size()) },
+                _ => System.oom(e),
+            }
+        }
+
+        unsafe fn realloc(
+            &mut self,
+            ptr: *mut u8,
+            layout: Layout,
+            new_layout: Layout,
+        ) -> Result<*mut u8, AllocErr> {
+            System.realloc(ptr, layout, new_layout)
+        }
+
+        unsafe fn alloc_zeroed(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
+            System.alloc_zeroed(layout)
+        }
+    }
+
+}
+
+#[cfg(feature = "oom_with_global_alloc")]
+#[global_allocator]
+static HEAP: global_alloc::GeckoHeap = global_alloc::GeckoHeap;
--- a/toolkit/xre/nsAppRunner.cpp
+++ b/toolkit/xre/nsAppRunner.cpp
@@ -226,16 +226,18 @@
 #include "sandboxPermissions.h"
 #endif
 #endif
 
 #ifdef MOZ_CODE_COVERAGE
 #include "mozilla/CodeCoverageHandler.h"
 #endif
 
+#include "mozilla/mozalloc_oom.h"
+
 extern uint32_t gRestartMode;
 extern void InstallSignalHandlers(const char *ProgramName);
 
 #define FILE_COMPATIBILITY_INFO NS_LITERAL_CSTRING("compatibility.ini")
 #define FILE_INVALIDATE_CACHES NS_LITERAL_CSTRING(".purgecaches")
 #define FILE_STARTUP_INCOMPLETE NS_LITERAL_STRING(".startup-incomplete")
 
 int    gArgc;
@@ -5364,8 +5366,15 @@ OverrideDefaultLocaleIfNeeded() {
 }
 
 void
 XRE_EnableSameExecutableForContentProc() {
   if (!PR_GetEnv("MOZ_SEPARATE_CHILD_PROCESS")) {
     mozilla::ipc::GeckoChildProcessHost::EnableSameExecutableForContentProc();
   }
 }
+
+// Because rust doesn't handle weak symbols, this function wraps the weak
+// malloc_handle_oom for it.
+extern "C" void
+GeckoHandleOOM(size_t size) {
+  mozalloc_handle_oom(size);
+}