Bug 1415689 - Add Clang 6 (pre) and use it for ASan builds. r?froydnj draft
authorChristian Holler <choller@mozilla.com>
Fri, 10 Nov 2017 10:14:26 +0100
changeset 699144 bc232d9a31279a5deb12b14aa156895829149f5e
parent 699134 f857506733963cfc23bae42249cfd8cf56155fb6
child 740549 b5eb2e758c80af1a4edb07df77c16b16cedf65ed
push id89478
push usercholler@mozilla.com
push dateThu, 16 Nov 2017 18:21:50 +0000
reviewersfroydnj
bugs1415689
milestone59.0a1
Bug 1415689 - Add Clang 6 (pre) and use it for ASan builds. r?froydnj MozReview-Commit-ID: 34rcG7gEswz
build/build-clang/clang-6-pre-linux64.json
build/build-clang/find_symbolizer_linux.patch
taskcluster/ci/build/linux.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/scripts/misc/build-clang-6-pre-linux.sh
new file mode 100644
--- /dev/null
+++ b/build/build-clang/clang-6-pre-linux64.json
@@ -0,0 +1,20 @@
+{
+    "llvm_revision": "317840",
+    "stages": "3",
+    "build_libcxx": true,
+    "build_type": "Release",
+    "assertions": false,
+    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
+    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
+    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
+    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
+    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
+    "python_path": "/usr/bin/python2.7",
+    "gcc_dir": "/builds/worker/workspace/build/src/gcc",
+    "cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
+    "cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
+    "as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
+    "patches": [
+      "find_symbolizer_linux.patch"
+    ]
+}
new file mode 100644
--- /dev/null
+++ b/build/build-clang/find_symbolizer_linux.patch
@@ -0,0 +1,58 @@
+We currently need this patch because ASan only searches PATH to find the
+llvm-symbolizer binary to symbolize ASan traces. On testing machines, this
+can be installed in PATH easily. However, for e.g. the ASan Nightly Project,
+where we ship an ASan build, including llvm-symbolizer, to the user, we
+cannot expect llvm-symbolizer to be on PATH. Instead, we should try to look
+it up next to the binary. This patch implements the functionality for Linux
+only until there is similar functionality provided upstream.
+
+diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_file.cc b/compiler-rt/lib/sanitizer_common/sanitizer_file.cc
+index cde54bf..8daade1 100644
+--- a/compiler-rt/lib/sanitizer_common/sanitizer_file.cc
++++ b/compiler-rt/lib/sanitizer_common/sanitizer_file.cc
+@@ -21,6 +21,10 @@
+ #include "sanitizer_common.h"
+ #include "sanitizer_file.h"
+ 
++#if SANITIZER_LINUX
++#include "sanitizer_posix.h"
++#endif
++
+ namespace __sanitizer {
+ 
+ void CatastrophicErrorWrite(const char *buffer, uptr length) {
+@@ -156,6 +160,34 @@ char *FindPathToBinary(const char *name) {
+     if (*end == '\0') break;
+     beg = end + 1;
+   }
++
++#if SANITIZER_LINUX
++  // If we cannot find the requested binary in PATH, we should try to locate
++  // it next to the binary, in case it is shipped with the build itself
++  // (e.g. llvm-symbolizer shipped with sanitizer build to symbolize on client.
++  if (internal_readlink("/proc/self/exe", buffer.data(), kMaxPathLength) < 0)
++    return nullptr;
++
++  uptr buf_len = internal_strlen(buffer.data());
++
++  /* Avoid using dirname() here */
++  while (buf_len > 0) {
++    if (buffer[buf_len - 1] == '/')
++      break;
++    buf_len--;
++  }
++
++  if (!buf_len)
++    return nullptr;
++
++  if (buf_len + name_len + 1 <= kMaxPathLength) {
++    internal_memcpy(&buffer[buf_len], name, name_len);
++    buffer[buf_len + name_len] = '\0';
++    if (FileExists(buffer.data()))
++      return internal_strdup(buffer.data());
++  }
++#endif
++
+   return nullptr;
+ }
+ 
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -474,17 +474,17 @@ linux64-asan/opt:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: asan-tc
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
-        - linux64-clang
+        - linux64-clang-6-pre
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
 
 linux64-asan-fuzzing/opt:
     description: "Linux64 Fuzzing Opt ASAN"
     index:
@@ -507,17 +507,17 @@ linux64-asan-fuzzing/opt:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: fuzzing-asan-tc
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
-        - linux64-clang-4
+        - linux64-clang-6-pre
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
 linux64-asan-reporter/opt:
     description: "Linux64 Opt ASAN Reporter Nightly"
     index:
         product: firefox
@@ -539,17 +539,17 @@ linux64-asan-reporter/opt:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: asan-reporter-tc
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
-        - linux64-clang-4
+        - linux64-clang-6-pre
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
 linux64-asan/debug:
     description: "Linux64 Debug ASAN"
     index:
         product: firefox
@@ -571,17 +571,17 @@ linux64-asan/debug:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: asan-tc-and-debug
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
-        - linux64-clang
+        - linux64-clang-6-pre
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -64,16 +64,38 @@ linux64-clang-5:
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-5-linux64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
+linux64-clang-6-pre:
+    description: "Clang 6 Pre toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(clang6p)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 7200
+    run:
+        using: toolchain-script
+        script: build-clang-6-pre-linux.sh
+        resources:
+            - 'build/build-clang/build-clang.py'
+            - 'build/build-clang/clang-6-pre-linux64.json'
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/clang.tar.xz
+    toolchains:
+        - linux64-gcc-4.9
+
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
         kind: build
         platform: toolchains/opt
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-clang-6-pre-linux.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building clang for Linux.
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-clang
+# |mach python| sets up a virtualenv for us!
+../../mach python ./build-clang.py -c clang-6-pre-linux64.json
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp clang.tar.* $UPLOAD_DIR