bug 1401647 - use a 64-bit Rust toolchain for win32 builds. r?build draft
authorTed Mielczarek <ted@mielczarek.org>
Thu, 14 Dec 2017 10:20:33 -0600
changeset 715762 748d4df6f17fbfe31d7102a455e086adb4afc57d
parent 715761 43bfc07ed2fade466aecd333bc4bb874c257938d
child 715763 0da027f54d27888a7e5191fc12846d7971a36217
push id94254
push userbmo:ted@mielczarek.org
push dateThu, 04 Jan 2018 16:52:13 +0000
reviewersbuild
bugs1401647
milestone59.0a1
bug 1401647 - use a 64-bit Rust toolchain for win32 builds. r?build We currently use a 32-bit Rust toolchain for win32 builds, but this can lead to OOM situations. This patch makes win32 builds use a 64-bit Rust toolchain, which requires a little bit of extra configuration because rustc needs to be able to find a link.exe that produces 64-bit binaries for building things like build scripts, which are host binaries. We will now generate a batch file that sets LIB to the paths to 64-bit libraries and invokes the x64-targeting link.exe, and add a section to the .cargo/config file to instruct cargo to use that batch file as the linker when producing 64-bit binaries. MozReview-Commit-ID: 9vKBbm7Gvra
.cargo/config.in
build/moz.build
build/moz.configure/rust.configure
build/win32/mozconfig.vs2017
build/win64/cargo-linker.bat.in
python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
taskcluster/ci/build/windows.yml
--- a/.cargo/config.in
+++ b/.cargo/config.in
@@ -4,8 +4,10 @@ replace-with = 'vendored-sources'
 
 [source."https://github.com/gankro/serde"]
 git = "https://github.com/gankro/serde"
 branch = "deserialize_from_enums3"
 replace-with = "vendored-sources"
 
 [source.vendored-sources]
 directory = '@top_srcdir@/third_party/rust'
+
+@WIN64_CARGO_LINKER_CONFIG@
--- a/build/moz.build
+++ b/build/moz.build
@@ -7,16 +7,18 @@
 with Files('**'):
     BUG_COMPONENT = ('Core', 'Build Config')
 
 # This cannot be named "build" because of bug 922191.
 SPHINX_TREES['buildsystem'] = 'docs'
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     DIRS += ['win32']
+    if CONFIG['WIN64_CARGO_LINKER']:
+        CONFIGURE_SUBST_FILES += ['win64/cargo-linker.bat']
 else:
     DIRS += ['unix']
 
 CRAMTEST_MANIFESTS += [
     'tests/cram/cram.ini',
 ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
--- a/build/moz.configure/rust.configure
+++ b/build/moz.configure/rust.configure
@@ -21,16 +21,17 @@ cargo = check_prog('CARGO', add_rustup_p
 @depends_if(rustc)
 @checking('rustc version', lambda info: info.version)
 def rustc_info(rustc):
     out = check_cmd_output(rustc, '--version', '--verbose').splitlines()
     info = dict((s.strip() for s in line.split(':', 1)) for line in out[1:])
     return namespace(
         version=Version(info.get('release', '0')),
         commit=info.get('commit-hash', 'unknown'),
+        host=info['host'],
     )
 
 
 @depends_if(cargo)
 @checking('cargo version', lambda info: info.version)
 @imports('re')
 def cargo_info(cargo):
     out = check_cmd_output(cargo, '--version', '--verbose').splitlines()
@@ -258,8 +259,44 @@ add_old_configure_assignment('RUSTC', ru
 add_old_configure_assignment('RUST_TARGET', rust_target_triple)
 
 # This option is separate from --enable-tests because Rust tests are particularly
 # expensive in terms of compile time (especially for code in libxul).
 option('--enable-rust-tests',
        help='Enable building of Rust tests, and build-time execution of them')
 
 set_config('MOZ_RUST_TESTS', depends('--enable-rust-tests')(lambda v: bool(v)))
+
+option(env='WIN64_LINK', nargs=1, help='Path to link.exe that targets win64')
+option(env='WIN64_LIB', nargs=1, help='Paths to libraries for the win64 linker')
+
+set_config('WIN64_LINK', depends('WIN64_LINK')(lambda x: x))
+set_config('WIN64_LIB', depends('WIN64_LIB')(lambda x: x))
+
+
+@depends(target, rustc_info, c_compiler, 'WIN64_LINK', 'WIN64_LIB')
+def win64_cargo_linker(target, rustc_info, compiler_info, link, lib):
+    # When we're building a 32-bit Windows build with a 64-bit rustc, we
+    # need to configure the linker it will use for host binaries (build scripts)
+    # specially because the compiler configuration we use for the build is for
+    # MSVC targeting 32-bit binaries.
+    if target.kernel == 'WINNT' and target.cpu == 'x86' and \
+       compiler_info.type in ('msvc', 'clang-cl') and \
+       rustc_info.host == 'x86_64-pc-windows-msvc' and link and lib:
+        return True
+
+
+set_config('WIN64_CARGO_LINKER', win64_cargo_linker)
+
+
+@depends(win64_cargo_linker, check_build_environment)
+@imports(_from='textwrap', _import='dedent')
+def win64_cargo_linker_config(linker, env):
+    if linker:
+        return dedent('''\
+        [target.x86_64-pc-windows-msvc]
+        linker = "{objdir}/build/win64/cargo-linker.bat"
+        '''.format(objdir=env.topobjdir))
+    # We want an empty string here so we don't leave the @ variable in the config file.
+    return ''
+
+
+set_config('WIN64_CARGO_LINKER_CONFIG', win64_cargo_linker_config)
--- a/build/win32/mozconfig.vs2017
+++ b/build/win32/mozconfig.vs2017
@@ -10,16 +10,19 @@ if [ -d "${VSPATH}" ]; then
     export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT"
     export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86"
 
     export PATH="${VSPATH}/VC/bin/Hostx86/x86:${VSPATH}/VC/bin/Hostx64/x86:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/DIA SDK/bin:${PATH}"
     export PATH="${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${PATH}"
 
     export INCLUDE="${VSPATH}/VC/include:${VSPATH}/VC/atlmfc/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um:${VSPATH}/SDK/Include/10.0.15063.0/winrt:${VSPATH}/DIA SDK/include"
     export LIB="${VSPATH}/VC/lib/x86:${VSPATH}/VC/atlmfc/lib/x86:${VSPATH}/SDK/Lib/10.0.15063.0/ucrt/x86:${VSPATH}/SDK/Lib/10.0.15063.0/um/x86:${VSPATH}/DIA SDK/lib"
+
+    export WIN64_LINK="${VSPATH}/VC/bin/Hostx64/x64/link.exe"
+    export WIN64_LIB="${VSPATH}/VC/lib/x64:${VSPATH}/VC/atlmfc/lib/x64:${VSPATH}/SDK/Lib/10.0.15063.0/ucrt/x64:${VSPATH}/SDK/Lib/10.0.15063.0/um/x64:${VSPATH}/DIA SDK/lib/amd64"
 fi
 
 . $topsrcdir/build/mozconfig.vs-common
 
 mk_export_correct_style WINDOWSSDKDIR
 mk_export_correct_style WIN32_REDIST_DIR
 mk_export_correct_style WIN_UCRT_REDIST_DIR
 mk_export_correct_style PATH
new file mode 100644
--- /dev/null
+++ b/build/win64/cargo-linker.bat.in
@@ -0,0 +1,3 @@
+set LIB=@WIN64_LIB@
+
+@WIN64_LINK@ %*
--- a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
+++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py
@@ -1415,17 +1415,17 @@ class OpenBSDToolchainTest(BaseToolchain
 class RustTest(BaseConfigureTest):
     def invoke_cargo(self, stdin, args):
         if args == ('--version', '--verbose'):
             return 0, 'cargo 2.0\nrelease: 2.0', ''
         raise NotImplementedError('unsupported arguments')
 
     def invoke_rustc(self, stdin, args):
         if args == ('--version', '--verbose'):
-            return 0, 'rustc 2.0\nrelease: 2.0', ''
+            return 0, 'rustc 2.0\nrelease: 2.0\nhost: x86_64-unknown-linux-gnu', ''
         if args == ('--print', 'target-list'):
             # Raw list returned by rustc version 1.19, + ios, which somehow
             # don't appear in the default list.
             # https://github.com/rust-lang/rust/issues/36156
             rust_targets = [
                 'aarch64-apple-ios',
                 'aarch64-linux-android',
                 'aarch64-unknown-freebsd',
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -17,18 +17,18 @@ win32/debug:
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/debug.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32/opt:
     description: "Win32 Opt"
     index:
         product: firefox
         job-name: win32-opt
     treeherder:
@@ -45,18 +45,18 @@ win32/opt:
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32-dmd/opt:
     description: "Win32 DMD Opt"
     index:
         product: firefox
         job-name: win32-dmd-opt
     treeherder:
@@ -76,18 +76,18 @@ win32-dmd/opt:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
         extra-config:
             mozconfig_variant: 'opt-dmd'
     run-on-projects: []
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32/pgo:
     description: "Win32 Opt PGO"
     index:
         product: firefox
         job-name: win32-pgo
     treeherder:
@@ -104,18 +104,18 @@ win32/pgo:
         options: [enable-pgo, append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64/debug:
     description: "Win64 Debug"
     index:
         product: firefox
         job-name: win64-debug
     treeherder:
@@ -227,18 +227,18 @@ win32-nightly/opt:
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
             - disable_signing.py
             - taskcluster_nightly.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-nightly/opt:
     description: "Win64 Nightly"
     index:
         product: firefox
         job-name: win64-opt
         type: nightly
@@ -321,18 +321,18 @@ win32-add-on-devel/opt:
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/addondevel.py
             - balrog/production.py
     run-on-projects: ['mozilla-beta', 'mozilla-release', 'mozilla-esr45']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-add-on-devel/opt:
     description: "Windows64 add-on-devel"
     index:
         product: firefox
         job-name: win64-add-on-devel
     treeherder:
@@ -409,18 +409,18 @@ win32-noopt/debug:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/noopt_debug.py
     run-on-projects: ['trunk', 'try']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32-rusttests/opt:
     description: "Win32 Opt Rust tests"
     index:
         product: firefox
         job-name: win32-rusttests-opt
     treeherder:
@@ -439,18 +439,18 @@ win32-rusttests/opt:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/releng_base_firefox.py
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/rusttests_opt.py
     run-on-projects: ['trunk', 'try']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-rusttests/opt:
     description: "Win64 Opt Rust tests"
     index:
         product: firefox
         job-name: win64-rusttests-opt
     treeherder:
@@ -595,18 +595,18 @@ win32-devedition-nightly/opt:
             - builds/taskcluster_base_windows.py
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
             - disable_signing.py
             - taskcluster_nightly.py
         custom-build-variant-cfg: devedition
     run-on-projects: ['mozilla-beta']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-devedition-nightly/opt:
     description: "Win64 Dev Edition Nightly"
     index:
         product: devedition
         job-name: win64-opt
         type: nightly