Bug 1386782 - Add toolchain tasks to build gn in automation. draft
authorChris Manchester <cmanchester@mozilla.com>
Tue, 25 Jul 2017 15:33:44 -0700
changeset 656132 2ec6ba44d8f50726d2f2af6c00b3aea61e495391
parent 655379 8f221407c6006ac1de529da77ecc18431dde3e99
child 729020 fa524bab9d8766e9d4618f76a276d900ef841843
push id77078
push usercmanchester@mozilla.com
push dateWed, 30 Aug 2017 19:08:36 +0000
bugs1386782
milestone57.0a1
Bug 1386782 - Add toolchain tasks to build gn in automation. MozReview-Commit-ID: 7dDwsQJnzRL
browser/config/tooltool-manifests/win32/gn-build.manifest
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/scripts/misc/build-gn-common.sh
taskcluster/scripts/misc/build-gn-linux.sh
taskcluster/scripts/misc/build-gn-macosx.sh
taskcluster/scripts/misc/build-gn-win32.sh
taskcluster/scripts/misc/gn.patch
new file mode 100644
--- /dev/null
+++ b/browser/config/tooltool-manifests/win32/gn-build.manifest
@@ -0,0 +1,26 @@
+[
+  {
+    "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
+    "size": 326656969,
+    "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7",
+    "algorithm": "sha512",
+    "filename": "vs2015u3.zip",
+    "unpack": true
+  },
+  {
+    "version": "Ninja 1.7.1",
+    "size": 184821,
+    "digest": "e4f9a1ae624a2630e75264ba37d396d9c7407d6e6aea3763056210ba6e1387908bd31cf4037a6a3661a418e86c4d2761e0c333e6a3bd0d66549d2b0d72d3f43b",
+    "algorithm": "sha512",
+    "filename": "ninja171.zip",
+    "unpack": true
+  },
+  {
+    "version": "MinGit-2.13.3-64-bit",
+    "size": 21482885,
+    "digest": "929bb3c07be8487ee519422a312bdbfeec8f4db4b62c49d02f9aad9fd2a66c0ee5fad63d2b06c8744c336dc9d50446fa4457897333ad17ffd783ecabd1e2ddbb",
+    "algorithm": "sha512",
+    "filename": "git.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -201,8 +201,30 @@ linux64-sccache:
     run:
         using: toolchain-script
         script: build-sccache.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/sccache2.tar.xz
     toolchains:
         - linux64-clang-3.9
+
+linux64-gn:
+    description: "gn toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(gn)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: build-gn-linux.sh
+        tooltool-downloads: public
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+            - 'taskcluster/scripts/misc/build-gn-common.sh'
+        toolchain-artifact: public/build/gn.tar.xz
+    toolchains:
+        - linux64-gcc-4.9
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -78,8 +78,33 @@ macosx64-cctools-port:
         script: build-cctools-port-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/cctools.tar.bz2
     toolchains:
         - linux64-cctools-port
         - linux64-clang-3.9
+
+macosx64-gn:
+    description: "gn toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TM(gn)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+    run:
+        using: toolchain-script
+        script: build-gn-macosx.sh
+        tooltool-downloads: internal
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+            - 'taskcluster/scripts/misc/build-gn-common.sh'
+        toolchain-artifact: public/build/gn.tar.xz
+    toolchains:
+        - linux64-cctools-port
+        - linux64-clang-3.9
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -107,8 +107,28 @@ win64-sccache:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
     run:
         using: toolchain-script
         script: build-sccache.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/sccache2.tar.bz2
+
+win32-gn:
+    description: "gn toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TW32(gn)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/gn-build.manifest"
+    run:
+        using: toolchain-script
+        script: build-gn-win32.sh
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+            - 'taskcluster/scripts/misc/build-gn-common.sh'
+        toolchain-artifact: public/build/gn.tar.bz2
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-gn-common.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+set -e -v
+
+# This is shared code for building GN.
+
+# Each is a recent commit from chromium's master branch.
+: CHROMIUM_REV           ${CHROMIUM_REV:=e6ba81e00ae835946e069e5bd80bd533b11d8442}
+: GTEST_REV              ${GTEST_REV:=6c5116014ce51ef3273d800cbf75fcef99e798c6}
+: CHROMIUM_SRC_REV       ${CHROMIUM_SRC_REV:=c338d43f49c0d72e69cd6e40eeaf4c0597dbdda1}
+
+
+git clone --no-checkout https://chromium.googlesource.com/chromium/src $WORKSPACE/gn-standalone
+cd $WORKSPACE/gn-standalone
+git checkout $CHROMIUM_SRC_REV
+
+git clone --no-checkout https://chromium.googlesource.com/chromium/chromium chromium_checkout
+cd chromium_checkout
+git checkout $CHROMIUM_REV
+mkdir -p ../third_party
+mv third_party/libevent ../third_party
+cd ..
+
+rm -rf testing
+mkdir testing
+cd testing
+git clone https://chromium.googlesource.com/chromium/testing/gtest
+cd gtest
+git checkout $GTEST_REV
+cd ../..
+
+cd tools/gn
+patch -p1 < $WORKSPACE/build/src/taskcluster/scripts/misc/gn.patch
+
+./bootstrap/bootstrap.py -s
+cd ../..
+
+STAGE=gn
+mkdir -p $UPLOAD_DIR $STAGE
+
+# At this point, the resulting binary is at:
+# $WORKSPACE/out/Release/gn
+if test "$MAC_CROSS" = "" -a "$(uname)" = "Linux"; then
+    strip out/Release/gn
+fi
+cp out/Release/gn $STAGE
+
+tar -acf gn.tar.$COMPRESS_EXT $STAGE
+cp gn.tar.$COMPRESS_EXT $UPLOAD_DIR
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-gn-linux.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e -v
+
+# This script is for building GN on Linux.
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+export CC=$WORKSPACE/build/src/gcc/bin/gcc
+export CXX=$WORKSPACE/build/src/gcc/bin/g++
+
+# Gn build scripts use #!/usr/bin/env python, which will be python 2.6 on
+# the worker and cause failures. Work around this by putting python2.7
+# ahead of it in PATH.
+mkdir -p $WORKSPACE/python_bin
+ln -s /usr/bin/python2.7 $WORKSPACE/python_bin/python
+export PATH=$WORKSPACE/python_bin:$PATH
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+. taskcluster/scripts/misc/build-gn-common.sh
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-gn-macosx.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+set -e -v
+
+# This script is for building GN.
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+CROSS_CCTOOLS_PATH=$WORKSPACE/build/src/cctools
+CROSS_SYSROOT=$WORKSPACE/build/src/MacOSX10.10.sdk
+
+export LD_LIBRARY_PATH=$WORKSPACE/build/src/clang/lib
+export CC=$WORKSPACE/build/src/clang/bin/clang
+export CXX=$WORKSPACE/build/src/clang/bin/clang++
+export AR=$WORKSPACE/build/src/clang/bin/clang
+export CFLAGS="-target x86_64-apple-darwin11 -mlinker-version=137 -B ${CROSS_CCTOOLS_PATH}/bin -isysroot ${CROSS_SYSROOT} -I${CROSS_SYSROOT}/usr/include -iframework ${CROSS_SYSROOT}/System/Library/Frameworks"
+export CXXFLAGS="-stdlib=libc++ ${CFLAGS}"
+export LDFLAGS="${CXXFLAGS} -Wl,-syslibroot,${CROSS_SYSROOT} -Wl,-dead_strip"
+
+# We patch tools/gn/bootstrap/bootstrap.py to detect this.
+export MAC_CROSS=1
+
+# Gn build scripts use #!/usr/bin/env python, which will be python 2.6 on
+# the worker and cause failures. Work around this by putting python2.7
+# ahead of it in PATH.
+mkdir -p $WORKSPACE/python_bin
+ln -s /usr/bin/python2.7 $WORKSPACE/python_bin/python
+export PATH=$WORKSPACE/python_bin:$PATH
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+# The ninja templates used to bootstrap gn have hard-coded references to
+# 'libtool', make sure we find the right one.
+ln -s $CROSS_CCTOOLS_PATH/bin/x86_64-apple-darwin11-libtool $CROSS_CCTOOLS_PATH/bin/libtool
+export PATH=$CROSS_CCTOOLS_PATH/bin:$PATH
+
+. taskcluster/scripts/misc/build-gn-common.sh
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-gn-win32.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e -v
+
+# This script is for building GN on Windows.
+
+WORKSPACE=$PWD
+UPLOAD_DIR=$WORKSPACE/public/build
+COMPRESS_EXT=bz2
+
+VSPATH="$WORKSPACE/build/src/vs2015u3"
+
+export INCLUDE="${VSPATH}/VC/include:${VSPATH}/VC/atlmfc/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um:${VSPATH}/SDK/Include/10.0.14393.0/winrt:${VSPATH}/DIA SDK/include"
+export LIB="${VSPATH}/VC/lib:${VSPATH}/VC/atlmfc/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86:${VSPATH}/DIA SDK/lib"
+
+export PATH="$WORKSPACE/build/src/ninja/bin:$PATH"
+export PATH="$WORKSPACE/build/src/mingw64/bin:$PATH"
+export PATH="${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/DIA SDK/bin:${PATH}"
+export PATH="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+. taskcluster/scripts/misc/build-gn-common.sh
new file mode 100644
--- /dev/null
+++ b/taskcluster/scripts/misc/gn.patch
@@ -0,0 +1,78 @@
+diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py
+index ff2ae57..7e12436 100755
+--- a/bootstrap/bootstrap.py
++++ b/bootstrap/bootstrap.py
+@@ -29,9 +29,10 @@ BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
+ GN_ROOT = os.path.dirname(BOOTSTRAP_DIR)
+ SRC_ROOT = os.path.dirname(os.path.dirname(GN_ROOT))
+ 
++is_mac_cross = os.environ.get('MAC_CROSS', False)
+ is_win = sys.platform.startswith('win')
+-is_linux = sys.platform.startswith('linux')
+-is_mac = sys.platform.startswith('darwin')
++is_linux = sys.platform.startswith('linux') and not is_mac_cross
++is_mac = sys.platform.startswith('darwin') or is_mac_cross
+ is_aix = sys.platform.startswith('aix')
+ is_posix = is_linux or is_mac or is_aix
+ 
+@@ -595,7 +596,6 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'base/synchronization/condition_variable_posix.cc',
+         'base/synchronization/lock_impl_posix.cc',
+         'base/synchronization/read_write_lock_posix.cc',
+-        'base/synchronization/waitable_event_posix.cc',
+         'base/sys_info_posix.cc',
+         'base/task_scheduler/task_tracker_posix.cc',
+         'base/threading/platform_thread_internal_posix.cc',
+@@ -603,10 +603,19 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'base/threading/thread_local_storage_posix.cc',
+         'base/threading/worker_pool_posix.cc',
+         'base/time/time_conversion_posix.cc',
++    ])
++
++    if not is_mac:
++      static_libraries['base']['sources'].extend([
++        'base/synchronization/waitable_event_posix.cc',
+         'base/time/time_exploded_posix.cc',
+         'base/time/time_now_posix.cc',
+-        'base/trace_event/heap_profiler_allocation_register_posix.cc',
++      ])
++
++    static_libraries['base']['sources'].extend([
++      'base/trace_event/heap_profiler_allocation_register_posix.cc',
+     ])
++
+     static_libraries['libevent'] = {
+         'sources': [
+             'base/third_party/libevent/buffer.c',
+@@ -685,6 +694,7 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'base/mac/call_with_eh_frame.cc',
+         'base/mac/call_with_eh_frame_asm.S',
+         'base/mac/foundation_util.mm',
++        'base/mac/mac_util.mm',
+         'base/mac/mach_logging.cc',
+         'base/mac/scoped_mach_port.cc',
+         'base/mac/scoped_mach_vm.cc',
+@@ -697,6 +707,7 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'base/process/process_iterator_mac.cc',
+         'base/process/process_metrics_mac.cc',
+         'base/strings/sys_string_conversions_mac.mm',
++        'base/synchronization/waitable_event_mac.cc',
+         'base/sys_info_mac.mm',
+         'base/time/time_mac.cc',
+         'base/threading/platform_thread_mac.mm',
+@@ -735,6 +746,7 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'base/memory/memory_pressure_monitor_win.cc',
+         'base/memory/shared_memory_handle_win.cc',
+         'base/memory/shared_memory_win.cc',
++        'base/memory/shared_memory_tracker.cc',
+         'base/message_loop/message_pump_win.cc',
+         'base/native_library_win.cc',
+         'base/power_monitor/power_monitor_device_source_win.cc',
+@@ -799,6 +811,7 @@ def write_gn_ninja(path, root_gen_dir, options):
+         'version.lib',
+         'winmm.lib',
+         'ws2_32.lib',
++        'shlwapi.lib',
+     ])
+ 
+   # we just build static libraries that GN needs