Bug 1433279 - Add native macOS host-utils repack toolchain job.
MozReview-Commit-ID: 2pEKRNyqV8h
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -12,8 +12,9 @@ transforms:
job-defaults:
run-on-projects: []
jobs-from:
- linux.yml
- macosx.yml
- windows.yml
+ - macosxn.yml
copy from taskcluster/ci/toolchain/macosx.yml
copy to taskcluster/ci/toolchain/macosxn.yml
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosxn.yml
@@ -1,113 +1,26 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
job-defaults:
- worker:
- docker-image: {in-tree: toolchain-build}
run-on-projects:
- trunk
- try
-macosx64-clang:
- description: "Clang toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(clang)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
- run:
- using: toolchain-script
- script: build-clang-macosx.sh
- tooltool-downloads: internal
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-macosx64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang.tar.xz
- toolchains:
- - linux64-cctools-port
- - linux64-clang-3.9
- - linux64-gcc-4.9
-
-macosx64-clang-tidy:
- description: "Clang-tidy build"
- index:
- product: static-analysis
- job-name: macosx64-clang-tidy
+macosx64-host-utils:
+ description: "host-utils repack toolchain build"
treeherder:
kind: build
platform: toolchains/opt
- symbol: TM(clang-tidy)
+ symbol: TM(host-utils)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: releng-hardware/gecko-t-osx-1010
worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+ max-run-time: 600
run:
using: toolchain-script
- script: build-clang-tidy-macosx.sh
+ script: repack-host-utils-macosx64.sh
tooltool-downloads: internal
resources:
- - 'build/clang-plugin/**'
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-tidy-macosx64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang-tidy.tar.xz
- toolchains:
- - linux64-cctools-port
- - linux64-clang-3.9
- - linux64-gcc-4.9
-
-macosx64-cctools-port:
- description: "cctools-port toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(cctools)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
- run:
- using: toolchain-script
- script: build-cctools-port-macosx.sh
- tooltool-downloads: internal
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/cctools.tar.bz2
- toolchains:
- - linux64-cctools-port
- - linux64-clang-3.9
-
-macosx64-gn:
- description: "gn toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(gn)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
- run:
- using: toolchain-script
- script: build-gn-macosx.sh
- tooltool-downloads: internal
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- - 'taskcluster/scripts/misc/build-gn-common.sh'
- toolchain-artifact: public/build/gn.tar.xz
- toolchains:
- - linux64-cctools-port
- - linux64-clang-3.9
+ - 'build/package/mac_osx/unpack-diskimage'
+ toolchain-artifact: public/build/host-utils-macosx64.tar.xz
copy from taskcluster/scripts/misc/repack-android-sdk-linux.sh
copy to taskcluster/scripts/misc/repack-host-utils-macosx64.sh
--- a/taskcluster/scripts/misc/repack-android-sdk-linux.sh
+++ b/taskcluster/scripts/misc/repack-host-utils-macosx64.sh
@@ -1,22 +1,34 @@
#!/bin/bash
set -x -e -v
-# This script is for fetching and repacking the Android SDK (for
-# Linux), the tools required to produce Android packages.
+# This script is for fetching, repacking, and code signing for macOS,
+# hostutils for Android (and other platforms). See
+# https://wiki.mozilla.org/Packaging_Android_host_utilities.
+
+UPLOAD_DIR=public/build
-WORKSPACE=$HOME/workspace
-UPLOAD_DIR=$HOME/project/gecko/android-sdk
+mkdir -p $UPLOAD_DIR
+
+curl -L -ounpack-diskimage "${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/build/package/mac_osx/unpack-diskimage"
+curl -L -otarget.tests.zip https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-60.0a1.en-US.mac.common.tests.zip
+curl -L -otarget.dmg https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-60.0a1.en-US.mac.dmg
-mkdir -p $HOME/artifacts $UPLOAD_DIR
+rm -rf bin
+unzip target.tests.zip "bin/*"
+mv bin host-utils
-# Populate /builds/worker/.mozbuild/android-sdk-linux.
-cd /builds/worker/workspace/build/src
-./mach python python/mozboot/mozboot/android.py --artifact-mode --no-interactive
+mkdir -p unpack
+/bin/bash unpack-diskimage target.dmg tmp unpack
+
+cp -R unpack/Firefox*.app/Contents/MacOS/* host-utils
+cp -R unpack/Firefox*.app/Contents/Resources/* host-utils
+
+find host-utils -type f -perm +111 -print | grep -v \\. | xargs codesign --force --deep --sign -
# It's nice to have the build logs include the state of the world upon
# completion.
-/builds/worker/.mozbuild/android-sdk-linux/tools/bin/sdkmanager --list
+ls -al host-utils
-tar cf - -C /builds/worker/.mozbuild android-sdk-linux | xz > $UPLOAD_DIR/android-sdk-linux.tar.xz
+tar cf - host-utils | xz > $UPLOAD_DIR/host-utils-macosx64.tar.xz
ls -al $UPLOAD_DIR
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -178,18 +178,73 @@ def docker_worker_toolchain(config, job,
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
-@run_job_using("generic-worker", "toolchain-script",
- schema=toolchain_run_schema, defaults=toolchain_defaults)
+def macosx_toolchain(config, job, taskdesc):
+ run = job['run']
+
+ worker = taskdesc['worker']
+
+ worker['artifacts'] = [{
+ 'path': r'public/build',
+ 'type': 'directory',
+ }]
+ worker['chain-of-trust'] = True
+
+ docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+ env = worker['env']
+ env.update({
+ 'MOZ_BUILD_DATE': config.params['moz_build_date'],
+ 'MOZ_SCM_LEVEL': config.params['level'],
+ 'MOZ_AUTOMATION': '1',
+ })
+
+ if run['script'].endswith('.py'):
+ raise NotImplementedError("Python scripts don't work on Mac OS X")
+
+ args = run.get('arguments', '')
+ if args:
+ args = ' ' + shell_quote(*args)
+
+ url = '${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/taskcluster/scripts/misc/%s' \
+ % run['script']
+
+ worker['command'] = [
+ [
+ 'bash', '-c',
+ 'pwd && '
+ 'find . && '
+ 'curl -o ./script.sh --retry 10 "{url}" && '
+ 'chmod +x ./script.sh &&'
+ './script.sh{args}'.format(
+ url=url, args=args)
+ ],
+ ]
+
+ attributes = taskdesc.setdefault('attributes', {})
+ attributes['toolchain-artifact'] = run['toolchain-artifact']
+ if 'toolchain-alias' in run:
+ attributes['toolchain-alias'] = run['toolchain-alias']
+
+ if not taskgraph.fast:
+ name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
+ add_optimization(
+ config, taskdesc,
+ cache_type=CACHE_TYPE,
+ cache_name=name,
+ digest_data=get_digest_data(config, run, taskdesc),
+ )
+
+
def windows_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = [{
'path': r'public\build',
'type': 'directory',
@@ -239,8 +294,21 @@ def windows_toolchain(config, job, taskd
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
+
+
+@run_job_using("generic-worker", "toolchain-script",
+ schema=toolchain_run_schema, defaults=toolchain_defaults)
+def generic_worker_toolchain(config, job, taskdesc):
+ os = taskdesc['worker']['os']
+ if os == 'windows':
+ return windows_toolchain(config, job, taskdesc)
+ elif os == 'macosx':
+ return macosx_toolchain(config, job, taskdesc)
+ else:
+ raise NotImplementedError("Toolchain jobs run using generic-worker "
+ "not implemented on OS: {}".format(os))