Bug 1421728 - Add a macosx64 fuzzing-asan build. r?froydnj,dustin draft
authorJesse Schwartzentruber <jdschwa@gmail.com>
Thu, 08 Feb 2018 17:16:41 -0500
changeset 755805 d03b4e431d0cb27dfccee72985bebf3c711d3874
parent 754870 e293877d13a5236119adb706c97c55ea9e11868b
push id99281
push userbmo:jschwartzentruber@mozilla.com
push dateThu, 15 Feb 2018 19:54:22 +0000
reviewersfroydnj, dustin
bugs1421728
milestone60.0a1
Bug 1421728 - Add a macosx64 fuzzing-asan build. r?froydnj,dustin MozReview-Commit-ID: DNNu4jyG50Z
browser/config/mozconfigs/linux32/debug-asan
browser/config/mozconfigs/linux32/nightly-asan
browser/config/mozconfigs/linux64/debug-asan
browser/config/mozconfigs/linux64/nightly-asan
browser/config/mozconfigs/linux64/nightly-asan-reporter
browser/config/mozconfigs/linux64/nightly-fuzzing-asan
browser/config/mozconfigs/macosx64/nightly-asan
browser/config/mozconfigs/macosx64/nightly-fuzzing-asan
build/unix/mozconfig.asan
build/unix/mozconfig.fuzzing
build/unix/rewrite_asan_dylib.py
taskcluster/ci/build/macosx.yml
testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/browser/config/mozconfigs/linux32/debug-asan
+++ b/browser/config/mozconfigs/linux32/debug-asan
@@ -4,16 +4,17 @@ ac_add_options --enable-debug
 ac_add_options --enable-optimize="-O1"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 # Enable Telemetry
 export MOZ_TELEMETRY_REPORTING=1
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
--- a/browser/config/mozconfigs/linux32/nightly-asan
+++ b/browser/config/mozconfigs/linux32/nightly-asan
@@ -3,16 +3,17 @@ ac_add_options --disable-debug
 ac_add_options --enable-optimize="-O2 -g"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Enable Telemetry
 export MOZ_TELEMETRY_REPORTING=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
--- a/browser/config/mozconfigs/linux64/debug-asan
+++ b/browser/config/mozconfigs/linux64/debug-asan
@@ -4,16 +4,17 @@ ac_add_options --enable-debug
 ac_add_options --enable-optimize="-O1"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 # Enable Telemetry
 export MOZ_TELEMETRY_REPORTING=1
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
--- a/browser/config/mozconfigs/linux64/nightly-asan
+++ b/browser/config/mozconfigs/linux64/nightly-asan
@@ -3,16 +3,17 @@ ac_add_options --disable-debug
 ac_add_options --enable-optimize="-O2 -gline-tables-only"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
 export MOZ_PKG_SPECIAL=asan
 
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/config/mozconfigs/linux64/nightly-asan-reporter
+++ b/browser/config/mozconfigs/linux64/nightly-asan-reporter
@@ -3,15 +3,16 @@ ac_add_options --disable-debug
 ac_add_options --enable-optimize="-O2 -gline-tables-only"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 ac_add_options --enable-address-sanitizer-reporter
 
 # Need this to prevent name conflicts with the normal nightly build packages
 export MOZ_PKG_SPECIAL=asan-reporter
 
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/config/mozconfigs/linux64/nightly-fuzzing-asan
+++ b/browser/config/mozconfigs/linux64/nightly-fuzzing-asan
@@ -7,17 +7,18 @@ MOZ_REQUIRE_SIGNING=0
 # Enable MOZ_ALLOW_LEGACY_EXTENSIONS
 ac_add_options "MOZ_ALLOW_LEGACY_EXTENSIONS=1"
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
-. $topsrcdir/build/unix/mozconfig.fuzzing
+. $topsrcdir/build/unix/mozconfig.asan
+ac_add_options --disable-elf-hack
 
 ac_add_options --enable-fuzzing
 unset MOZ_STDCXX_COMPAT
 
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
--- a/browser/config/mozconfigs/macosx64/nightly-asan
+++ b/browser/config/mozconfigs/macosx64/nightly-asan
@@ -11,9 +11,14 @@ export MOZ_TELEMETRY_REPORTING=1
 # Package js shell.
 export MOZ_PACKAGE_JSSHELL=1
 
 # Need this to prevent name conflicts with the normal nightly build packages
 # Before mozconfig.common so we can test for asan builds there
 export MOZ_PKG_SPECIAL=asan
 
 . "$topsrcdir/build/macosx/mozconfig.common"
+
+# This is disabled by mozconfig.asan and reenabled by mozconfig.common.
+# Ensure it is disabled since it conflicts with ASan.
+ac_add_options --disable-crashreporter
+
 . "$topsrcdir/build/mozconfig.common.override"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/macosx64/nightly-fuzzing-asan
@@ -0,0 +1,2 @@
+. "$topsrcdir/browser/config/mozconfigs/macosx64/nightly-asan"
+ac_add_options --enable-fuzzing
--- a/build/unix/mozconfig.asan
+++ b/build/unix/mozconfig.asan
@@ -17,12 +17,11 @@ fi
 ac_add_options --enable-address-sanitizer
 
 # Mandatory options required for ASan builds (both on Linux and Mac)
 export MOZ_DEBUG_SYMBOLS=1
 ac_add_options --enable-debug-symbols
 ac_add_options --disable-install-strip
 ac_add_options --disable-jemalloc
 ac_add_options --disable-crashreporter
-ac_add_options --disable-elf-hack
 ac_add_options --disable-profiling
 
 . "$topsrcdir/build/unix/mozconfig.stdcxx"
deleted file mode 100644
--- a/build/unix/mozconfig.fuzzing
+++ /dev/null
@@ -1,28 +0,0 @@
-MOZ_AUTOMATION_L10N_CHECK=0
-
-. "$topsrcdir/build/mozconfig.common"
-
-# Use Clang as specified in manifest
-export CC="$topsrcdir/clang/bin/clang -fgnu89-inline"
-export CXX="$topsrcdir/clang/bin/clang++"
-export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer"
-
-# Use a newer binutils, from the tooltool gcc package, if it's there
-if [ -e "$topsrcdir/gcc/bin/ld" ]; then
-    export CC="$CC -B $topsrcdir/gcc/bin"
-    export CXX="$CXX -B $topsrcdir/gcc/bin"
-fi
-
-# Enable ASan specific code and build workarounds
-ac_add_options --enable-address-sanitizer
-
-# Mandatory options required for ASan builds (both on Linux and Mac)
-export MOZ_DEBUG_SYMBOLS=1
-ac_add_options --enable-debug-symbols
-ac_add_options --disable-install-strip
-ac_add_options --disable-jemalloc
-ac_add_options --disable-crashreporter
-ac_add_options --disable-elf-hack
-ac_add_options --disable-profiling
-
-. "$topsrcdir/build/unix/mozconfig.stdcxx"
--- a/build/unix/rewrite_asan_dylib.py
+++ b/build/unix/rewrite_asan_dylib.py
@@ -40,57 +40,65 @@ def resolve_rpath(filename):
         if currentCmd == 'LC_RPATH':
             pathMatch = re.match(r'^\s+path (.*) \(offset \d+\)', line)
             if pathMatch is not None:
                 path = pathMatch.group(1)
                 if os.path.isdir(path):
                     return path
 
     sys.stderr.write('@rpath could not be resolved from %s\n' % filename)
-    exit(1)
+    sys.exit(1)
 
 def scan_directory(path):
     dylibCopied = False
 
     for root, subdirs, files in os.walk(path):
         for filename in files:
             filename = os.path.join(root, filename)
 
             # Skip all files that aren't either dylibs or executable
             if not (filename.endswith('.dylib') or os.access(filename, os.X_OK)):
                 continue
 
             try:
                 otoolOut = subprocess.check_output([substs['OTOOL'], '-L', filename])
-            except:
+            except Exception:
                 # Errors are expected on non-mach executables, ignore them and continue
                 continue
 
             for line in otoolOut.splitlines():
-                if line.find(DYLIB_NAME) != -1:
+                if DYLIB_NAME in line:
                     absDylibPath = line.split()[0]
 
                     # Don't try to rewrite binaries twice
-                    if absDylibPath.find('@executable_path/') == 0:
+                    if absDylibPath.startswith('@executable_path/'):
                         continue
 
                     if not dylibCopied:
-                        if absDylibPath.find('@rpath/') == 0:
+                        if absDylibPath.startswith('@rpath/'):
                             rpath = resolve_rpath(filename)
                             copyDylibPath = absDylibPath.replace('@rpath', rpath)
                         else:
                             copyDylibPath = absDylibPath
 
-                        # Copy the runtime once to the main directory, which is passed
-                        # as the argument to this function.
-                        shutil.copy(copyDylibPath, path)
+                        if os.path.isfile(copyDylibPath):
+                            # Copy the runtime once to the main directory, which is passed
+                            # as the argument to this function.
+                            shutil.copy(copyDylibPath, path)
 
-                        # Now rewrite the library itself
-                        subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-id', '@executable_path/' + DYLIB_NAME, os.path.join(path, DYLIB_NAME)])
-                        dylibCopied = True
+                            # Now rewrite the library itself
+                            subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-id', '@executable_path/' + DYLIB_NAME, os.path.join(path, DYLIB_NAME)])
+                            dylibCopied = True
+                        else:
+                            sys.stderr.write('dylib path in %s was not found at: %s\n' % (filename, copyDylibPath))
 
                     # Now use install_name_tool to rewrite the path in our binary
-                    subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-change', absDylibPath, '@executable_path/' + DYLIB_NAME, filename])
+                    relpath = '' if path == root else os.path.relpath(path, root) + '/'
+                    subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-change', absDylibPath, '@executable_path/' + relpath + DYLIB_NAME, filename])
                     break
 
+    if not dylibCopied:
+        sys.stderr.write('%s could not be found\n' % DYLIB_NAME)
+        sys.exit(1)
+
 if __name__ == '__main__':
     for d in sys.argv[1:]:
         scan_directory(d)
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -58,16 +58,49 @@ macosx64/opt:
         - linux64-cctools-port
         - linux64-clang
         - linux64-hfsplus
         - linux64-libdmg
         - linux64-llvm-dsymutil
         - linux64-rust-macos
         - linux64-sccache
 
+macosx64-asan-fuzzing/opt:
+    description: "MacOS X x64 Cross-compile Fuzzing ASAN"
+    index:
+        product: firefox
+        job-name: macosx64-fuzzing-asan-opt
+    treeherder:
+        platform: osx-cross/asan
+        symbol: Bof
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        max-run-time: 36000
+        env:
+            PERFHERDER_EXTRA_OPTIONS: asan-fuzzing
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets build update]
+        config:
+            - builds/releng_base_firefox.py
+            - builds/releng_base_mac_64_cross_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        custom-build-variant-cfg: cross-fuzzing-asan
+        tooltool-downloads: internal
+    toolchains:
+        - linux64-cctools-port
+        - linux64-clang-6-pre-macosx-cross
+        - linux64-hfsplus
+        - linux64-libdmg
+        - linux64-llvm-dsymutil
+        - linux64-rust-macos
+        - linux64-sccache
+
 macosx64-dmd/opt:
     description: "MacOS X x64 DMD Cross-compile"
     index:
         product: firefox
         job-name: macosx64-dmd-opt
     treeherder:
         platform: osx-10-10-dmd/opt
         symbol: Bdmd
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py
@@ -0,0 +1,31 @@
+import os
+
+config = {
+    'default_actions': [
+        'clobber',
+        'build',
+    ],
+    'stage_platform': 'macosx64-fuzzing-asan',
+    'publish_nightly_en_US_routes': False,
+    'build_type': 'asan',
+    'platform_supports_post_upload_to_latest': False,
+    'enable_signing': False,
+    #### 64 bit build specific #####
+    'env': {
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'DISPLAY': ':2',
+        'HG_SHARE_BASE_DIR': '/builds/hg-shared',
+        'MOZ_OBJDIR': '%(abs_obj_dir)s',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': '/builds/tooltool_cache',
+        'TOOLTOOL_HOME': '/builds',
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'LC_ALL': 'C',
+        'ASAN_OPTIONS': 'detect_leaks=0',
+        ## 64 bit specific
+        'PATH': '/usr/local/bin:/usr/lib64/ccache:/bin:\
+/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
+/tools/python27-mercurial/bin:/home/cltbld/bin',
+    },
+    'mozconfig_variant': 'nightly-fuzzing-asan',
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -409,16 +409,17 @@ class BuildOptionParser(object):
         'asan-tc': 'builds/releng_sub_%s_configs/%s_asan_tc.py',
         'asan-reporter-tc': 'builds/releng_sub_%s_configs/%s_asan_reporter_tc.py',
         'fuzzing-asan-tc': 'builds/releng_sub_%s_configs/%s_fuzzing_asan_tc.py',
         'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py',
         'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py',
         'cross-debug-searchfox': 'builds/releng_sub_%s_configs/%s_cross_debug_searchfox.py',
         'cross-debug-artifact': 'builds/releng_sub_%s_configs/%s_cross_debug_artifact.py',
         'cross-noopt-debug': 'builds/releng_sub_%s_configs/%s_cross_noopt_debug.py',
+        'cross-fuzzing-asan': 'builds/releng_sub_%s_configs/%s_cross_fuzzing_asan.py',
         'cross-artifact': 'builds/releng_sub_%s_configs/%s_cross_artifact.py',
         'debug': 'builds/releng_sub_%s_configs/%s_debug.py',
         'fuzzing-debug': 'builds/releng_sub_%s_configs/%s_fuzzing_debug.py',
         'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py',
         'asan-tc-and-debug': 'builds/releng_sub_%s_configs/%s_asan_tc_and_debug.py',
         'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py',
         'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
         'source': 'builds/releng_sub_%s_configs/%s_source.py',