Bug 1183613 - Cross compile universal OSX builds in Taskcluster; r?ted,froydnj draft
authorMike Shal <mshal@mozilla.com>
Fri, 21 Oct 2016 13:54:10 -0400
changeset 440538 22a1c87ddbe1f92cbda21192f1dddff991b607ba
parent 439734 51750761f2c61c64cf0553f6cb5fefd4999d3bc0
child 537399 c6c4a5470806d495c65dbf4cb1a7820bafaab65d
push id36254
push userbmo:mshal@mozilla.com
push dateThu, 17 Nov 2016 18:57:54 +0000
reviewersted, froydnj
bugs1183613
milestone53.0a1
Bug 1183613 - Cross compile universal OSX builds in Taskcluster; r?ted,froydnj MozReview-Commit-ID: HNTqiVF9gov
browser/config/mozconfigs/macosx-universal/nightly
build/autoconf/toolchain.m4
build/macosx/universal/mozconfig.common
js/src/devtools/Instruments.cpp
js/src/jit/shared/Assembler-shared.h
js/src/jit/x86/Assembler-x86.cpp
js/src/jit/x86/Assembler-x86.h
js/src/jit/x86/CodeGenerator-x86.cpp
js/src/wasm/WasmBaselineCompile.cpp
python/mozbuild/mozpack/unify.py
taskcluster/ci/build/macosx.yml
testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
toolkit/mozapps/installer/packager.py
--- a/browser/config/mozconfigs/macosx-universal/nightly
+++ b/browser/config/mozconfigs/macosx-universal/nightly
@@ -1,15 +1,19 @@
 . "$topsrcdir/browser/config/mozconfigs/macosx-universal/common-opt"
 
 ac_add_options --disable-install-strip
 ac_add_options --enable-verify-mar
 ac_add_options --enable-profiling
 ac_add_options --enable-instruments
-ac_add_options --enable-dtrace
+
+# Cross-universal builds fail when dtrace is enabled
+if test `uname -s` != Linux; then
+  ac_add_options --enable-dtrace
+fi
 
 if test "${MOZ_UPDATE_CHANNEL}" = "nightly"; then
 ac_add_options --with-macbundlename-prefix=Firefox
 fi
 
 ac_add_options --with-branding=browser/branding/nightly
 
 . "$topsrcdir/build/mozconfig.rust"
--- a/build/autoconf/toolchain.m4
+++ b/build/autoconf/toolchain.m4
@@ -78,16 +78,17 @@ case "${TOOLCHAIN_PREFIX}" in
 esac
 AC_PROG_CC
 AC_PROG_CXX
 
 AC_CHECK_PROGS(RANLIB, "${TOOLCHAIN_PREFIX}ranlib", :)
 AC_CHECK_PROGS(AR, "${TOOLCHAIN_PREFIX}ar", :)
 AC_CHECK_PROGS(AS, "${TOOLCHAIN_PREFIX}as", :)
 AC_CHECK_PROGS(LD, "${TOOLCHAIN_PREFIX}ld", :)
+AC_CHECK_PROGS(LIPO, "${TOOLCHAIN_PREFIX}lipo", :)
 AC_CHECK_PROGS(STRIP, "${TOOLCHAIN_PREFIX}strip", :)
 AC_CHECK_PROGS(WINDRES, "${TOOLCHAIN_PREFIX}windres", :)
 AC_CHECK_PROGS(OTOOL, "${TOOLCHAIN_PREFIX}otool", :)
 AC_CHECK_PROGS(OBJCOPY, "${TOOLCHAIN_PREFIX}objcopy", :)
 PATH=$_SAVE_PATH
 ])
 
 AC_DEFUN([MOZ_CXX11],
--- a/build/macosx/universal/mozconfig.common
+++ b/build/macosx/universal/mozconfig.common
@@ -1,21 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 mk_add_options MOZ_UNIFY_BDATE=1
 
-DARWIN_VERSION=`uname -r`
+DARWIN_VERSION=10
 ac_add_app_options i386 --target=i386-apple-darwin$DARWIN_VERSION
 ac_add_app_options x86_64 --target=x86_64-apple-darwin$DARWIN_VERSION
 ac_add_app_options i386 --with-unify-dist=../x86_64/dist
 ac_add_app_options x86_64 --with-unify-dist=../i386/dist
 
-ac_add_options --with-macos-sdk=/Developer/SDKs/MacOSX10.7.sdk
+if ! test `uname -s` = Linux; then
+  # Cross-universal builds already do the equivalent of this by setting -isysroot directly
+  ac_add_options --with-macos-sdk=/Developer/SDKs/MacOSX10.7.sdk
+fi
 
 . $topsrcdir/build/macosx/mozconfig.common
 
 # $MOZ_BUILD_APP is only defined when sourced by configure.  That's not a
 # problem, because the variables it affects only need to be set for
 # configure.
 if test -n "$MOZ_BUILD_APP" ; then
 if test "$MOZ_BUILD_APP" = "i386" -o "$MOZ_BUILD_APP" = "x86_64"; then
@@ -30,22 +33,22 @@ if test "$MOZ_BUILD_APP" = "i386" -o "$M
 
   # It's not strictly necessary to specify -arch during native builds, but it
   # makes the merged about:buildconfig easier to follow, and it reduces
   # conditionalized differences between builds.
   CC="$CC -arch $TARGET_CPU"
   CXX="$CXX -arch $TARGET_CPU"
 
   # These must be set for cross builds, and don't hurt straight builds.
-  RANLIB=ranlib
-  AR=ar
+  RANLIB="${TOOLCHAIN_PREFIX}ranlib"
+  AR="${TOOLCHAIN_PREFIX}ar"
   AS=$CC
   LD=ld
   STRIP="strip"
-  OTOOL="otool"
+  OTOOL="${TOOLCHAIN_PREFIX}otool"
 
   # Each per-CPU build should be entirely oblivious to the fact that a
   # universal binary will be produced.  The exception is packager.mk, which
   # needs to know to look for universal bits when building the .dmg.
   UNIVERSAL_BINARY=1
 
   export CC CXX HOST_CC HOST_CXX RANLIB AR AS LD STRIP OTOOL
 fi
--- a/js/src/devtools/Instruments.cpp
+++ b/js/src/devtools/Instruments.cpp
@@ -1,13 +1,14 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "Instruments.h"
+#include "mozilla/Attributes.h"
 
 #ifdef __APPLE__
 
 #include <dlfcn.h>
 #include <CoreFoundation/CoreFoundation.h>
 #include <unistd.h>
 
 // There are now 2 paths to the DTPerformanceSession framework. We try to load
@@ -37,17 +38,17 @@ typedef bool (*DTPerformanceSessionSaveF
 namespace Instruments {
 
 static const int kSamplingInterval = 20; // microseconds
 
 template<typename T>
 class AutoReleased
 {
 public:
-  AutoReleased(T aTypeRef) : mTypeRef(aTypeRef)
+  MOZ_IMPLICIT AutoReleased(T aTypeRef) : mTypeRef(aTypeRef)
   {
   }
   ~AutoReleased()
   {
     if (mTypeRef) {
       CFRelease(mTypeRef);
     }
   }
--- a/js/src/jit/shared/Assembler-shared.h
+++ b/js/src/jit/shared/Assembler-shared.h
@@ -502,17 +502,17 @@ class CodeOffsetJump
 #ifdef JS_SMALL_BRANCH
     CodeOffsetJump(size_t offset, size_t jumpTableIndex)
         : offset_(offset), jumpTableIndex_(jumpTableIndex)
     {}
     size_t jumpTableIndex() const {
         return jumpTableIndex_;
     }
 #else
-    CodeOffsetJump(size_t offset) : offset_(offset) {}
+    explicit CodeOffsetJump(size_t offset) : offset_(offset) {}
 #endif
 
     CodeOffsetJump() {
         mozilla::PodZero(this);
     }
 
     size_t offset() const {
         return offset_;
--- a/js/src/jit/x86/Assembler-x86.cpp
+++ b/js/src/jit/x86/Assembler-x86.cpp
@@ -66,17 +66,17 @@ Assembler::executableCopy(uint8_t* buffe
 }
 
 class RelocationIterator
 {
     CompactBufferReader reader_;
     uint32_t offset_;
 
   public:
-    RelocationIterator(CompactBufferReader& reader)
+    explicit RelocationIterator(CompactBufferReader& reader)
       : reader_(reader)
     { }
 
     bool read() {
         if (!reader_.more())
             return false;
         offset_ = reader_.readUnsigned();
         return true;
--- a/js/src/jit/x86/Assembler-x86.h
+++ b/js/src/jit/x86/Assembler-x86.h
@@ -155,24 +155,24 @@ static_assert(CodeAlignment % SimdMemory
 static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
   "Stack alignment should be larger than any of the alignments which are used for "
   "spilled values.  Thus it should be larger than the alignment for SIMD accesses.");
 
 static const uint32_t WasmStackAlignment = SimdMemoryAlignment;
 
 struct ImmTag : public Imm32
 {
-    ImmTag(JSValueTag mask)
+    explicit ImmTag(JSValueTag mask)
       : Imm32(int32_t(mask))
     { }
 };
 
 struct ImmType : public ImmTag
 {
-    ImmType(JSValueType type)
+    explicit ImmType(JSValueType type)
       : ImmTag(JSVAL_TYPE_TO_TAG(type))
     { }
 };
 
 static const Scale ScalePointer = TimesFour;
 
 } // namespace jit
 } // namespace js
--- a/js/src/jit/x86/CodeGenerator-x86.cpp
+++ b/js/src/jit/x86/CodeGenerator-x86.cpp
@@ -731,34 +731,34 @@ CodeGeneratorX86::visitWasmStoreGlobalVa
 namespace js {
 namespace jit {
 
 class OutOfLineTruncate : public OutOfLineCodeBase<CodeGeneratorX86>
 {
     LTruncateDToInt32* ins_;
 
   public:
-    OutOfLineTruncate(LTruncateDToInt32* ins)
+    explicit OutOfLineTruncate(LTruncateDToInt32* ins)
       : ins_(ins)
     { }
 
     void accept(CodeGeneratorX86* codegen) {
         codegen->visitOutOfLineTruncate(this);
     }
     LTruncateDToInt32* ins() const {
         return ins_;
     }
 };
 
 class OutOfLineTruncateFloat32 : public OutOfLineCodeBase<CodeGeneratorX86>
 {
     LTruncateFToInt32* ins_;
 
   public:
-    OutOfLineTruncateFloat32(LTruncateFToInt32* ins)
+    explicit OutOfLineTruncateFloat32(LTruncateFToInt32* ins)
       : ins_(ins)
     { }
 
     void accept(CodeGeneratorX86* codegen) {
         codegen->visitOutOfLineTruncateFloat32(this);
     }
     LTruncateFToInt32* ins() const {
         return ins_;
--- a/js/src/wasm/WasmBaselineCompile.cpp
+++ b/js/src/wasm/WasmBaselineCompile.cpp
@@ -248,27 +248,27 @@ class BaseCompiler
 #if defined(JS_CODEGEN_X64)
     typedef ScratchRegisterScope ScratchI32;
 #elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
     class ScratchI32
     {
 # ifdef DEBUG
         BaseCompiler& bc;
       public:
-        ScratchI32(BaseCompiler& bc) : bc(bc) {
+        explicit ScratchI32(BaseCompiler& bc) : bc(bc) {
             MOZ_ASSERT(!bc.scratchRegisterTaken());
             bc.setScratchRegisterTaken(true);
         }
         ~ScratchI32() {
             MOZ_ASSERT(bc.scratchRegisterTaken());
             bc.setScratchRegisterTaken(false);
         }
 # else
       public:
-        ScratchI32(BaseCompiler& bc) {}
+        explicit ScratchI32(BaseCompiler& bc) {}
 # endif
         operator Register() const {
 # ifdef JS_CODEGEN_X86
             return ScratchRegX86;
 # else
             return ScratchRegARM;
 # endif
         }
--- a/python/mozbuild/mozpack/unify.py
+++ b/python/mozbuild/mozpack/unify.py
@@ -17,16 +17,17 @@ from mozpack.executables import (
 from mozpack.mozjar import JarReader
 from mozpack.errors import errors
 from tempfile import mkstemp
 import mozpack.path as mozpath
 import struct
 import os
 import re
 import subprocess
+import buildconfig
 from collections import OrderedDict
 
 # Regular expressions for unifying install.rdf
 FIND_TARGET_PLATFORM = re.compile(r"""
     <(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform>       # The targetPlatform tag, with any namespace
     (?P<platform>[^<]*)                             # The actual platform value
     </(?P=ns)?targetPlatform>                       # The closing tag
     """, re.X)
@@ -75,17 +76,18 @@ class UnifiedExecutableFile(BaseFile):
         assert isinstance(dest, basestring)
         tmpfiles = []
         try:
             for e in self._executables:
                 fd, f = mkstemp()
                 os.close(fd)
                 tmpfiles.append(f)
                 e.copy(f, skip_if_older=False)
-            subprocess.call(['lipo', '-create'] + tmpfiles + ['-output', dest])
+            lipo = buildconfig.substs.get('LIPO') or 'lipo'
+            subprocess.call([lipo, '-create'] + tmpfiles + ['-output', dest])
         finally:
             for f in tmpfiles:
                 os.unlink(f)
 
 
 class UnifiedFinder(BaseFinder):
     '''
     Helper to get unified BaseFile instances from two distinct trees on the
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -39,8 +39,32 @@ macosx64/opt:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
             - builds/releng_base_mac_64_cross_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
+
+macosx64-universal/opt:
+    description: "MacOS X Universal Cross-compile"
+    index:
+        product: firefox
+        job-name: macosx64-opt
+    treeherder:
+        platform: osx-10-7/opt
+        symbol: tc(Bu)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+    run:
+        using: mozharness
+        actions: [get-secrets build generate-build-stats update]
+        config:
+            - builds/releng_base_mac_64_cross_builds.py
+            - balrog/production.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: cross-universal
+        tooltool-downloads: internal
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py
@@ -0,0 +1,4 @@
+config = {
+    'objdir': 'obj-firefox/x86_64',
+    'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/nightly',
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -344,16 +344,17 @@ class BuildOptionParser(object):
     # platform/bits
     build_variants = {
         'add-on-devel': 'builds/releng_sub_%s_configs/%s_add-on-devel.py',
         'asan': 'builds/releng_sub_%s_configs/%s_asan.py',
         'asan-tc': 'builds/releng_sub_%s_configs/%s_asan_tc.py',
         'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py',
         'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py',
         'cross-opt': 'builds/releng_sub_%s_configs/%s_cross_opt.py',
+        'cross-universal': 'builds/releng_sub_%s_configs/%s_cross_universal.py',
         'debug': 'builds/releng_sub_%s_configs/%s_debug.py',
         'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py',
         'asan-tc-and-debug': 'builds/releng_sub_%s_configs/%s_asan_tc_and_debug.py',
         'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py',
         'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
         'source': 'builds/releng_sub_%s_configs/%s_source.py',
         'api-15-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_15_gradle_dependencies.py',
         'api-15': 'builds/releng_sub_%s_configs/%s_api_15.py',
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -320,17 +320,17 @@ def main():
         def is_native(path):
             path = os.path.abspath(path)
             return platform.machine() in mozpath.split(path)
 
         # Invert args.unify and args.source if args.unify points to the
         # native architecture.
         args.source, args.unify = sorted([args.source, args.unify],
                                          key=is_native, reverse=True)
-        if is_native(args.source):
+        if is_native(args.source) and not buildconfig.substs['CROSS_COMPILE']:
             launcher.tooldir = args.source
     elif not buildconfig.substs['CROSS_COMPILE']:
         launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
 
     with errors.accumulate():
         finder_args = dict(
             minify=args.minify,
             minify_js=args.minify_js,