Bug 1374940 - Add artifact paths to toolchain jobs definitions. r?dustin
The toolchain jobs produce artifacts that are going to be used by other
jobs, but there is no reliable way for the decision task to know the
name of those artifacts. So we make their definition required in the
toolchain job definitions.
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -16,16 +16,17 @@ linux64-clang:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
run:
using: toolchain-script
script: build-clang-linux.sh
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/clang.tar.xz
linux64-clang-tidy:
description: "Clang-tidy build"
index:
product: static-analysis
job-name: linux64-clang-tidy
treeherder:
kind: build
@@ -40,16 +41,17 @@ linux64-clang-tidy:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
run:
using: toolchain-script
script: build-clang-tidy-linux.sh
resources:
- 'build/clang-plugin/**'
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/clang-tidy.tar.xz
linux64-gcc:
description: "GCC toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(gcc)
tier: 1
@@ -57,16 +59,17 @@ linux64-gcc:
worker:
docker-image: {in-tree: desktop-build}
max-run-time: 36000
run:
using: toolchain-script
script: build-gcc-linux.sh
resources:
- 'build/unix/build-gcc/**'
+ toolchain-artifact: public/gcc.tar.xz
linux64-binutils:
description: "Binutils toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(binutil)
tier: 1
@@ -74,16 +77,17 @@ linux64-binutils:
worker:
docker-image: {in-tree: desktop-build}
max-run-time: 36000
run:
using: toolchain-script
script: build-binutils-linux.sh
resources:
- 'build/unix/build-binutils/**'
+ toolchain-artifact: public/binutils.tar.xz
linux64-cctools-port:
description: "cctools-port toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(cctools)
tier: 1
@@ -93,16 +97,17 @@ linux64-cctools-port:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
run:
using: toolchain-script
script: build-cctools-port.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/cctools.tar.xz
linux64-hfsplus:
description: "hfsplus toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(hfs+)
tier: 1
@@ -113,23 +118,25 @@ linux64-hfsplus:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
run:
using: toolchain-script
script: build-hfsplus-linux.sh
resources:
- 'build/unix/build-hfsplus/**'
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/hfsplus-tools.tar.xz
linux64-libdmg:
description: "libdmg-hfsplus toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(libdmg-hfs+)
tier: 1
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
docker-image: {in-tree: desktop-build}
max-run-time: 36000
run:
using: toolchain-script
script: build-libdmg-hfsplus.sh
+ toolchain-artifact: public/dmg.tar.xz
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -17,16 +17,17 @@ macosx64-clang:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-clang-macosx.sh
tooltool-downloads: internal
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/clang.tar.xz
macosx64-clang-tidy:
description: "Clang-tidy build"
index:
product: static-analysis
job-name: macosx64-clang-tidy
treeherder:
kind: build
@@ -42,16 +43,17 @@ macosx64-clang-tidy:
run:
using: toolchain-script
script: build-clang-tidy-macosx.sh
tooltool-downloads: internal
resources:
- 'build/clang-plugin/**'
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/clang-tidy.tar.xz
macosx64-cctools-port:
description: "cctools-port toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TM(cctools)
tier: 1
@@ -62,8 +64,9 @@ macosx64-cctools-port:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-cctools-port-macosx.sh
tooltool-downloads: internal
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
+ toolchain-artifact: public/cctools.tar.bz2
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -15,16 +15,17 @@ win32-clang-cl:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang32-windows.sh
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+ toolchain-artifact: public/build/clang.tar.bz2
win64-clang-cl:
description: "Clang-cl toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TW64(clang-cl)
tier: 2
@@ -34,16 +35,17 @@ win64-clang-cl:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang64-windows.sh
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+ toolchain-artifact: public/build/clang.tar.bz2
win32-clang-tidy:
description: "Clang-tidy toolchain build"
index:
product: static-analysis
job-name: win32-clang-tidy
treeherder:
kind: build
@@ -56,16 +58,17 @@ win32-clang-tidy:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang-tidy32-windows.sh
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.bz2
win64-clang-tidy:
description: "Clang-tidy toolchain build"
index:
product: static-analysis
job-name: win64-clang-tidy
treeherder:
kind: build
@@ -78,8 +81,9 @@ win64-clang-tidy:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
run:
using: toolchain-script
script: build-clang-tidy64-windows.sh
resources:
- 'build/build-clang/**'
- 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+ toolchain-artifact: public/build/clang-tidy.tar.bz2
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -159,9 +159,13 @@ specific locale involved. Currently this
signed
======
Signals that the output of this task contains signed artifacts.
repackage_type
==============
This is the type of repackage. Can be ``repackage`` or
-``repacakge_signing``.
+``repackage_signing``.
+
+toolchain-artifact
+==================
+For toolchain jobs, this is the path to the artifact for that toolchain.
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -34,16 +34,19 @@ toolchain_run_schema = Schema({
False,
'public',
'internal',
),
# Paths/patterns pointing to files that influence the outcome of a
# toolchain build.
Optional('resources'): [basestring],
+
+ # Path to the artifact produced by the toolchain job
+ Required('toolchain-artifact'): basestring,
})
def add_optimizations(config, run, taskdesc):
files = list(run.get('resources', []))
# This file
files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
# The script
@@ -125,16 +128,19 @@ def docker_worker_toolchain(config, job,
'--',
'bash',
'-c',
'cd /home/worker && '
'./workspace/build/src/taskcluster/scripts/misc/{}'.format(
run['script'])
]
+ attributes = taskdesc.setdefault('attributes', {})
+ attributes['toolchain-artifact'] = run['toolchain-artifact']
+
add_optimizations(config, run, taskdesc)
@run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
def windows_toolchain(config, job, taskdesc):
run = job['run']
taskdesc['run-on-projects'] = ['trunk', 'try']
@@ -176,9 +182,12 @@ def windows_toolchain(config, job, taskd
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
]
+ attributes = taskdesc.setdefault('attributes', {})
+ attributes['toolchain-artifact'] = run['toolchain-artifact']
+
add_optimizations(config, run, taskdesc)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -210,17 +210,16 @@ task_description_schema = Schema({
# command in the docker image
Optional('command'): [taskref_or_string],
# the maximum time to run, in seconds
Required('max-run-time'): int,
# the exit status code that indicates the task should be retried
Optional('retry-exit-status'): int,
-
}, {
Required('implementation'): 'generic-worker',
Required('os'): Any('windows', 'macosx'),
# see http://schemas.taskcluster.net/generic-worker/v1/payload.json
# and https://docs.taskcluster.net/reference/workers/generic-worker/payload
# command is a list of commands to run, sequentially
# on Windows, each command is a string, on OS X and Linux, each command is