Bug 1374940 - Add artifact paths to toolchain jobs definitions. r?dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 21 Jun 2017 15:13:35 +0900
changeset 612572 069daa0c1f0f7ca83a4ebe8cd07a0757d196ae7c
parent 612571 cb64b3cd99195b3de90a27c6a95ce85ba6ec1cbc
child 612573 a84ecddca2306f46d78569c6e4d7103cace0d836
push id69542
push userbmo:mh+mozilla@glandium.org
push dateThu, 20 Jul 2017 21:02:57 +0000
reviewersdustin
bugs1374940
milestone56.0a1
Bug 1374940 - Add artifact paths to toolchain jobs definitions. r?dustin The toolchain jobs produce artifacts that are going to be used by other jobs, but there is no reliable way for the decision task to know the name of those artifacts. So we make their definition required in the toolchain job definitions.
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/job/toolchain.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -16,16 +16,17 @@ linux64-clang:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-linux.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang.tar.xz
 
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
         kind: build
@@ -40,16 +41,17 @@ linux64-clang-tidy:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy-linux.sh
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang-tidy.tar.xz
 
 linux64-gcc:
     description: "GCC toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(gcc)
         tier: 1
@@ -57,16 +59,17 @@ linux64-gcc:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-gcc-linux.sh
         resources:
             - 'build/unix/build-gcc/**'
+        toolchain-artifact: public/gcc.tar.xz
 
 linux64-binutils:
     description: "Binutils toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
@@ -74,16 +77,17 @@ linux64-binutils:
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-binutils-linux.sh
         resources:
             - 'build/unix/build-binutils/**'
+        toolchain-artifact: public/binutils.tar.xz
 
 linux64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(cctools)
         tier: 1
@@ -93,16 +97,17 @@ linux64-cctools-port:
         max-run-time: 36000
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/cctools.tar.xz
 
 linux64-hfsplus:
     description: "hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
@@ -113,23 +118,25 @@ linux64-hfsplus:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/clang.manifest"
     run:
         using: toolchain-script
         script: build-hfsplus-linux.sh
         resources:
             - 'build/unix/build-hfsplus/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/hfsplus-tools.tar.xz
 
 linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(libdmg-hfs+)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-libdmg-hfsplus.sh
+        toolchain-artifact: public/dmg.tar.xz
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -17,16 +17,17 @@ macosx64-clang:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-clang-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang.tar.xz
 
 macosx64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: macosx64-clang-tidy
     treeherder:
         kind: build
@@ -42,16 +43,17 @@ macosx64-clang-tidy:
     run:
         using: toolchain-script
         script: build-clang-tidy-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/clang-tidy.tar.xz
 
 macosx64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(cctools)
         tier: 1
@@ -62,8 +64,9 @@ macosx64-cctools-port:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-cctools-port-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/cctools.tar.bz2
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -15,16 +15,17 @@ win32-clang-cl:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+        toolchain-artifact: public/build/clang.tar.bz2
 
 win64-clang-cl:
     description: "Clang-cl toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW64(clang-cl)
         tier: 2
@@ -34,16 +35,17 @@ win64-clang-cl:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang64-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+        toolchain-artifact: public/build/clang.tar.bz2
 
 win32-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win32-clang-tidy
     treeherder:
         kind: build
@@ -56,16 +58,17 @@ win32-clang-tidy:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
+        toolchain-artifact: public/build/clang-tidy.tar.bz2
 
 win64-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win64-clang-tidy
     treeherder:
         kind: build
@@ -78,8 +81,9 @@ win64-clang-tidy:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
     run:
         using: toolchain-script
         script: build-clang-tidy64-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
+        toolchain-artifact: public/build/clang-tidy.tar.bz2
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -159,9 +159,13 @@ specific locale involved. Currently this
 
 signed
 ======
 Signals that the output of this task contains signed artifacts.
 
 repackage_type
 ==============
 This is the type of repackage. Can be ``repackage`` or 
-``repacakge_signing``.
+``repackage_signing``.
+
+toolchain-artifact
+==================
+For toolchain jobs, this is the path to the artifact for that toolchain.
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -34,16 +34,19 @@ toolchain_run_schema = Schema({
         False,
         'public',
         'internal',
     ),
 
     # Paths/patterns pointing to files that influence the outcome of a
     # toolchain build.
     Optional('resources'): [basestring],
+
+    # Path to the artifact produced by the toolchain job
+    Required('toolchain-artifact'): basestring,
 })
 
 
 def add_optimizations(config, run, taskdesc):
     files = list(run.get('resources', []))
     # This file
     files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
     # The script
@@ -125,16 +128,19 @@ def docker_worker_toolchain(config, job,
         '--',
         'bash',
         '-c',
         'cd /home/worker && '
         './workspace/build/src/taskcluster/scripts/misc/{}'.format(
             run['script'])
     ]
 
+    attributes = taskdesc.setdefault('attributes', {})
+    attributes['toolchain-artifact'] = run['toolchain-artifact']
+
     add_optimizations(config, run, taskdesc)
 
 
 @run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
 def windows_toolchain(config, job, taskdesc):
     run = job['run']
     taskdesc['run-on-projects'] = ['trunk', 'try']
 
@@ -176,9 +182,12 @@ def windows_toolchain(config, job, taskd
 
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
         ' '.join(hg_command),
         # do something intelligent.
         r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
     ]
 
+    attributes = taskdesc.setdefault('attributes', {})
+    attributes['toolchain-artifact'] = run['toolchain-artifact']
+
     add_optimizations(config, run, taskdesc)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -210,17 +210,16 @@ task_description_schema = Schema({
         # command in the docker image
         Optional('command'): [taskref_or_string],
 
         # the maximum time to run, in seconds
         Required('max-run-time'): int,
 
         # the exit status code that indicates the task should be retried
         Optional('retry-exit-status'): int,
-
     }, {
         Required('implementation'): 'generic-worker',
         Required('os'): Any('windows', 'macosx'),
         # see http://schemas.taskcluster.net/generic-worker/v1/payload.json
         # and https://docs.taskcluster.net/reference/workers/generic-worker/payload
 
         # command is a list of commands to run, sequentially
         # on Windows, each command is a string, on OS X and Linux, each command is