Bug 1397847: make toolchain definitions a bit more generic; r?glandium
This makes the toolchain definitions a little more generic and useful for other
purposes, but includes a transform that turns them back into toolchain jobs.
MozReview-Commit-ID: 7NRrru9B3iU
new file mode 100644
--- /dev/null
+++ b/build/toolchains/README.md
@@ -0,0 +1,8 @@
+# Toolchain Definitions
+
+This directory contains definitions of toolchains used in automation.
+
+The content of these files are turned into Taskclutser tasks by the taskgraph
+generation, and in particular by the toolchain kind, defined in
+`taskcluster/ci/toolchain`. The format of each stanza in the files here is
+defined in `taskcluster/taskgraph/transforms/toolchain.py.
--- a/build/toolchains/linux.yml
+++ b/build/toolchains/linux.yml
@@ -1,230 +1,152 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
linux64-clang-3.9:
description: "Clang 3.9 toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(clang3.9)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(clang3.9)
+ run-on: linux
run:
using: toolchain-script
script: build-clang-3.9-linux.sh
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-3.9-linux64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-alias: linux64-clang
- toolchain-artifact: public/build/clang.tar.xz
+ resources:
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-3.9-linux64.json'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-alias: linux64-clang
+ toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-4.9
linux64-clang-4:
description: "Clang 4 toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(clang4)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(clang4)
+ run-on: linux
run:
using: toolchain-script
script: build-clang-4-linux.sh
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-4-linux64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang.tar.xz
+ resources:
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-4-linux64.json'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-4.9
linux64-clang-tidy:
description: "Clang-tidy build"
index:
product: static-analysis
job-name: linux64-clang-tidy
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(clang-tidy)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(clang-tidy)
+ run-on: linux
run:
using: toolchain-script
script: build-clang-tidy-linux.sh
- resources:
- - 'build/clang-plugin/**'
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-tidy-linux64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang-tidy.tar.xz
+ resources:
+ - 'build/clang-plugin/**'
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-tidy-linux64.json'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.xz
toolchains:
- linux64-gcc-4.9
linux64-gcc-4.9:
description: "GCC 4.9 toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(gcc4.9)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(gcc4.9)
+ run-on: linux
run:
using: toolchain-script
script: build-gcc-4.9-linux.sh
- resources:
- - 'build/unix/build-gcc/build-gcc.sh'
- toolchain-artifact: public/build/gcc.tar.xz
+ resources:
+ - 'build/unix/build-gcc/build-gcc.sh'
+ toolchain-artifact: public/build/gcc.tar.xz
linux64-gcc-6:
description: "GCC 6 toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(gcc6)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(gcc6)
+ run-on: linux
run:
using: toolchain-script
script: build-gcc-6-linux.sh
- resources:
- - 'build/unix/build-gcc/build-gcc.sh'
- toolchain-alias: linux64-gcc
- toolchain-artifact: public/build/gcc.tar.xz
+ resources:
+ - 'build/unix/build-gcc/build-gcc.sh'
+ toolchain-alias: linux64-gcc
+ toolchain-artifact: public/build/gcc.tar.xz
linux64-binutils:
description: "Binutils toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(binutil)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(binutil)
+ run-on: linux
run:
using: toolchain-script
script: build-binutils-linux.sh
- resources:
- - 'build/unix/build-binutils/build-binutils.sh'
- toolchain-artifact: public/build/binutils.tar.xz
+ resources:
+ - 'build/unix/build-binutils/build-binutils.sh'
+ toolchain-artifact: public/build/binutils.tar.xz
linux64-cctools-port:
description: "cctools-port toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(cctools)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(cctools)
+ run-on: linux
run:
using: toolchain-script
script: build-cctools-port.sh
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/cctools.tar.xz
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/cctools.tar.xz
toolchains:
- linux64-clang-3.9
linux64-hfsplus:
description: "hfsplus toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(hfs+)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(hfs+)
+ run-on: linux
run:
using: toolchain-script
script: build-hfsplus-linux.sh
- resources:
- - 'build/unix/build-hfsplus/build-hfsplus.sh'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/hfsplus-tools.tar.xz
+ resources:
+ - 'build/unix/build-hfsplus/build-hfsplus.sh'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/hfsplus-tools.tar.xz
toolchains:
- linux64-clang-3.9
linux64-libdmg:
description: "libdmg-hfsplus toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(libdmg-hfs+)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(libdmg-hfs+)
+ run-on: linux
run:
using: toolchain-script
script: build-libdmg-hfsplus.sh
- toolchain-artifact: public/build/dmg.tar.xz
+ toolchain-artifact: public/build/dmg.tar.xz
linux64-sccache:
description: "sccache toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(sccache)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+ symbol: TL(sccache)
+ run-on: linux
+ tooltool-manifest: "browser/config/tooltool-manifests/linux64/releng.manifest"
run:
using: toolchain-script
script: build-sccache.sh
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/sccache2.tar.xz
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/sccache2.tar.xz
toolchains:
- linux64-clang-3.9
linux64-gn:
description: "gn toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TL(gn)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
+ symbol: TL(gn)
+ run-on: linux
run:
using: toolchain-script
script: build-gn-linux.sh
tooltool-downloads: public
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- - 'taskcluster/scripts/misc/build-gn-common.sh'
- toolchain-artifact: public/build/gn.tar.xz
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ - 'taskcluster/scripts/misc/build-gn-common.sh'
+ toolchain-artifact: public/build/gn.tar.xz
toolchains:
- linux64-gcc-4.9
--- a/build/toolchains/macosx.yml
+++ b/build/toolchains/macosx.yml
@@ -1,110 +1,78 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
macosx64-clang:
description: "Clang toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(clang)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+ symbol: TM(clang)
+ run-on: macosx-cross
+ tooltool-manifest: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-clang-macosx.sh
tooltool-downloads: internal
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-macosx64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang.tar.xz
+ resources:
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-macosx64.json'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-cctools-port
- linux64-clang-3.9
- linux64-gcc-4.9
macosx64-clang-tidy:
description: "Clang-tidy build"
index:
product: static-analysis
job-name: macosx64-clang-tidy
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(clang-tidy)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+ symbol: TM(clang-tidy)
+ run-on: macosx-cross
+ tooltool-manifest: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-clang-tidy-macosx.sh
tooltool-downloads: internal
- resources:
- - 'build/clang-plugin/**'
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-tidy-macosx64.json'
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/clang-tidy.tar.xz
+ resources:
+ - 'build/clang-plugin/**'
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-tidy-macosx64.json'
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.xz
toolchains:
- linux64-cctools-port
- linux64-clang-3.9
- linux64-gcc-4.9
macosx64-cctools-port:
description: "cctools-port toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(cctools)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+ symbol: TM(cctools)
+ run-on: macosx-cross
+ tooltool-manifest: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-cctools-port-macosx.sh
tooltool-downloads: internal
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/cctools.tar.bz2
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/cctools.tar.bz2
toolchains:
- linux64-cctools-port
- linux64-clang-3.9
macosx64-gn:
description: "gn toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TM(gn)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
- worker:
- docker-image: {in-tree: desktop-build}
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+ symbol: TM(gn)
+ run-on: macosx-cross
+ tooltool-manifest: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-gn-macosx.sh
tooltool-downloads: internal
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- - 'taskcluster/scripts/misc/build-gn-common.sh'
- toolchain-artifact: public/build/gn.tar.xz
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ - 'taskcluster/scripts/misc/build-gn-common.sh'
+ toolchain-artifact: public/build/gn.tar.xz
toolchains:
- linux64-cctools-port
- linux64-clang-3.9
--- a/build/toolchains/windows.yml
+++ b/build/toolchains/windows.yml
@@ -1,134 +1,94 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
win32-clang-cl:
description: "Clang-cl toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW32(clang-cl)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
+ symbol: TW32(clang-cl)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang32-windows.sh
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-win32.json'
- - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
- toolchain-artifact: public/build/clang.tar.bz2
+ resources:
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-win32.json'
+ - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+ toolchain-artifact: public/build/clang.tar.bz2
win64-clang-cl:
description: "Clang-cl toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW64(clang-cl)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
+ symbol: TW64(clang-cl)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang64-windows.sh
- resources:
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-win64.json'
- - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
- toolchain-artifact: public/build/clang.tar.bz2
+ resources:
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-win64.json'
+ - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+ toolchain-artifact: public/build/clang.tar.bz2
win32-clang-tidy:
description: "Clang-tidy toolchain build"
+ tier: 2
index:
product: static-analysis
job-name: win32-clang-tidy
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW32(clang-tidy)
- tier: 2
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
+ symbol: TW32(clang-tidy)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang-tidy32-windows.sh
- resources:
- - 'build/clang-plugin/**'
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-tidy-win32.json'
- - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
- toolchain-artifact: public/build/clang-tidy.tar.bz2
+ resources:
+ - 'build/clang-plugin/**'
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-tidy-win32.json'
+ - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.bz2
win64-clang-tidy:
description: "Clang-tidy toolchain build"
+ tier: 2
index:
product: static-analysis
job-name: win64-clang-tidy
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW64(clang-tidy)
- tier: 2
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
+ symbol: TW64(clang-tidy)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang-tidy64-windows.sh
- resources:
- - 'build/clang-plugin/**'
- - 'build/build-clang/build-clang.py'
- - 'build/build-clang/clang-tidy-win32.json'
- - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
- toolchain-artifact: public/build/clang-tidy.tar.bz2
+ resources:
+ - 'build/clang-plugin/**'
+ - 'build/build-clang/build-clang.py'
+ - 'build/build-clang/clang-tidy-win32.json'
+ - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.bz2
win64-sccache:
description: "sccache toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW64(sccache)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
+ symbol: TW64(sccache)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
run:
using: toolchain-script
script: build-sccache.sh
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- toolchain-artifact: public/build/sccache2.tar.bz2
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/build/sccache2.tar.bz2
win32-gn:
description: "gn toolchain build"
- treeherder:
- kind: build
- platform: toolchains/opt
- symbol: TW32(gn)
- tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
- worker:
- max-run-time: 36000
- env:
- TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/gn-build.manifest"
+ symbol: TW32(gn)
+ run-on: windows
+ tooltool-manifest: "browser/config/tooltool-manifests/win32/gn-build.manifest"
run:
using: toolchain-script
script: build-gn-win32.sh
- resources:
- - 'taskcluster/scripts/misc/tooltool-download.sh'
- - 'taskcluster/scripts/misc/build-gn-common.sh'
- toolchain-artifact: public/build/gn.tar.bz2
+ resources:
+ - 'taskcluster/scripts/misc/tooltool-download.sh'
+ - 'taskcluster/scripts/misc/build-gn-common.sh'
+ toolchain-artifact: public/build/gn.tar.bz2
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -1,15 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.transform:loader
transforms:
+ - taskgraph.transforms.toolchain:transforms
- taskgraph.transforms.try_job:transforms
- taskgraph.transforms.use_toolchains:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- /build/toolchains/linux.yml
- /build/toolchains/macosx.yml
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -5,17 +5,17 @@
Support for running toolchain-building jobs via dedicated scripts
"""
from __future__ import absolute_import, print_function, unicode_literals
import hashlib
from taskgraph.util.schema import Schema
-from voluptuous import Optional, Required, Any
+from voluptuous import Required, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_tc_vcs_cache,
docker_worker_add_gecko_vcs_env_vars,
docker_worker_add_public_artifacts,
docker_worker_add_tooltool,
support_vcs_checkout,
@@ -34,32 +34,21 @@ toolchain_run_schema = Schema({
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
'public',
'internal',
),
-
- # Paths/patterns pointing to files that influence the outcome of a
- # toolchain build.
- Optional('resources'): [basestring],
-
- # Path to the artifact produced by the toolchain job
- Required('toolchain-artifact'): basestring,
-
- # An alias that can be used instead of the real toolchain job name in
- # the toolchains list for build jobs.
- Optional('toolchain-alias'): basestring,
})
def add_optimizations(config, run, taskdesc):
- files = list(run.get('resources', []))
+ files = [] # fixed in later commits..
# This file
files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
# The script
files.append('taskcluster/scripts/misc/{}'.format(run['script']))
# Tooltool manifest if any is defined:
tooltool_manifest = taskdesc['worker']['env'].get('TOOLTOOL_MANIFEST')
if tooltool_manifest:
files.append(tooltool_manifest)
@@ -126,21 +115,16 @@ def docker_worker_toolchain(config, job,
'--',
'bash',
'-c',
'cd /builds/worker && '
'./workspace/build/src/taskcluster/scripts/misc/{}'.format(
run['script'])
]
- attributes = taskdesc.setdefault('attributes', {})
- attributes['toolchain-artifact'] = run['toolchain-artifact']
- if 'toolchain-alias' in run:
- attributes['toolchain-alias'] = run['toolchain-alias']
-
add_optimizations(config, run, taskdesc)
@run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
def windows_toolchain(config, job, taskdesc):
run = job['run']
taskdesc['run-on-projects'] = ['trunk', 'try']
@@ -174,13 +158,12 @@ def windows_toolchain(config, job, taskd
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
]
attributes = taskdesc.setdefault('attributes', {})
- attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
add_optimizations(config, run, taskdesc)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/toolchain.py
@@ -0,0 +1,121 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from voluptuous import Optional, Required, Any
+from taskgraph.util.schema import (
+ validate_schema,
+ Schema,
+)
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.transforms.job.toolchain import toolchain_run_schema
+
+logger = logging.getLogger(__name__)
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+toolchain_description_schema = Schema({
+ # Supplied by the task loader
+ Optional('name'): basestring,
+ Optional('job-from'): task_description_schema['job-from'],
+
+ # treeherder information
+ Required('tier', default=1): int,
+ Required('symbol'): basestring,
+ Required('description'): basestring,
+
+ # where to run builds of this toolchain. (note, 'macosx-cross' runs on linux)
+ Required('run-on'): Any('linux', 'macosx-cross', 'windows'),
+
+ # tooltool manifest to use (sets $TOOLTOOL_MANIFEST when building)
+ Optional('tooltool-manifest'): basestring,
+
+ # Path to the artifact produced by the toolchain job. This becomes a task
+ # attribute
+ Required('toolchain-artifact'): basestring,
+
+ # An alias that can be used instead of the real toolchain job name in
+ # the toolchains list for build jobs. This becomes a task attribute
+ Optional('toolchain-alias'): basestring,
+
+ # See taskcluster/taskgraph/transforms/task.py
+ Optional('index'): task_description_schema['index'],
+
+ # See taskcluster/taskgraph/transforms/job/toolchain.py
+ Required('run'): toolchain_run_schema,
+
+ # Toolchains on which this toolchain depends; see
+ # taskcluster/taskgraph/transform/use_toolchains.py
+ Required('toolchains', default=[]): [basestring],
+
+ # Paths/patterns pointing to files that influence the outcome of a
+ # toolchain build.
+ Optional('resources'): [basestring],
+})
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def validate(config, jobs):
+ for job in jobs:
+ yield validate_schema(toolchain_description_schema, job,
+ "In toolchain {!r}:".format(job.get('name', job.get('label'))))
+
+
+@transforms.add
+def toolchain_descr_to_job(config, toolchain_descriptions):
+ """Convert a toolchain description (in `/build/toolchains`) into a job
+ description with run.using toolchain-script. (Handled in the job transform
+ taskcluster/taskgraph/transforms/jobs/toolchain.py)"""
+ for tcd in toolchain_descriptions:
+
+ job = {}
+
+ job['name'] = tcd['name']
+ job['job-from'] = tcd['job-from']
+ job['description'] = tcd['description']
+ job['run'] = tcd['run']
+ job['toolchains'] = tcd['toolchains']
+ if 'index' in tcd:
+ job['index'] = tcd['index']
+
+ job['treeherder'] = {
+ 'kind': 'build',
+ 'platform': 'toolchains/opt',
+ 'tier': tcd.get('tier', 1),
+ 'symbol': tcd['symbol'],
+ }
+
+ run_on = tcd['run-on']
+ if run_on == 'linux':
+ job['worker-type'] = 'aws-provisioner-v1/gecko-{level}-b-linux'
+ job['worker'] = {
+ 'docker-image': {'in-tree': 'desktop-build'},
+ 'max-run-time': 36000,
+ }
+ elif run_on == 'macosx-cross':
+ job['worker-type'] = 'aws-provisioner-v1/gecko-{level}-b-macosx64'
+ job['worker'] = {
+ 'docker-image': {'in-tree': 'desktop-build'},
+ 'max-run-time': 36000,
+ }
+ elif run_on == 'windows':
+ job['worker-type'] = 'aws-provisioner-v1/gecko-{level}-b-win2012'
+ job['worker'] = {'max-run-time': 36000}
+
+ tooltool_manifest = tcd.pop('tooltool-manifest', None)
+ if tooltool_manifest:
+ job['worker']['env'] = {'TOOLTOOL_MANIFEST': tooltool_manifest}
+
+ job['attributes'] = {'toolchain-artifact': tcd['toolchain-artifact']}
+ if 'toolchain-alias' in tcd:
+ job['attributes']['toolchain-alias'] = tcd['toolchain-alias']
+
+ yield job
--- a/taskcluster/taskgraph/transforms/use_toolchains.py
+++ b/taskcluster/taskgraph/transforms/use_toolchains.py
@@ -26,19 +26,18 @@ def use_toolchains(config, jobs):
dict[key] = value
# Toolchain jobs can depend on other toolchain jobs, but we don't have full
# tasks for them, since they're being transformed. So scan the jobs list in
# that case, otherwise, use the list of tasks for the kind dependencies.
if config.kind == 'toolchain':
jobs = list(jobs)
for job in jobs:
- run = job.get('run', {})
- get_attribute(artifacts, job['name'], run, 'toolchain-artifact')
- get_attribute(aliases_by_job, job['name'], run, 'toolchain-alias')
+ get_attribute(artifacts, job['name'], job['attributes'], 'toolchain-artifact')
+ get_attribute(aliases_by_job, job['name'], job['attributes'], 'toolchain-alias')
else:
for task in config.kind_dependencies_tasks:
if task.kind != 'toolchain':
continue
name = task.label.replace('%s-' % task.kind, '')
get_attribute(artifacts, name, task.attributes, 'toolchain-artifact')
get_attribute(aliases_by_job, name, task.attributes, 'toolchain-alias')