Bug 1345863: set the job_try_name for all tasks governed by -j; r?gps draft
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 28 Apr 2017 11:12:31 -0400
changeset 576336 94ba7c2bf3e6330a657b735bc17cfd7812dc6399
parent 576335 2d5adb9d53a2bc85eb9da1ff27752bf577e94ccc
child 628157 e52825558228281f9ca299ecd01b450fd8bb2f7e
push id58318
push userdmitchell@mozilla.com
push dateThu, 11 May 2017 16:12:32 +0000
reviewersgps
bugs1345863
milestone55.0a1
Bug 1345863: set the job_try_name for all tasks governed by -j; r?gps We have a `unittest_try_name` for all unit test tasks, and similarly a `talos_try_name`. The `-j` flag controls tasks known as "jobs" (although the word has dozens of other meanings, too). Some of those set `job_try_name`, but others do not and have special-case support in `try_option_syntax.py`. With this change, all "jobs" set `job_try_name` and the special-case is removed. MozReview-Commit-ID: 9hvW7wBIl2B
taskcluster/ci/android-stuff/kind.yml
taskcluster/ci/toolchain/kind.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/taskgraph/transforms/try_job.py
taskcluster/taskgraph/try_option_syntax.py
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -4,25 +4,23 @@
 
 # The name of this kind should suggest it's not meant to be permanent.  This is
 # a temporary place to generate these tasks in Bug 1286075 until they are
 # rewritten in a better way.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
+   - taskgraph.transforms.try_job:transforms
    - taskgraph.transforms.android_stuff:transforms
    - taskgraph.transforms.task:transforms
 
 jobs:
     android-api-15-gradle-dependencies:
         description: "Android armv7 API 15+ gradle dependencies"
-        attributes:
-            build_platform: android-api-15-gradle-dependencies
-            build_type: opt
         index:
             product: mobile
             job-name: android-api-15-gradle-dependencies-opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             kind: other
             tier: 2
             symbol: tc(Deps)
@@ -61,19 +59,16 @@ jobs:
         optimizations:
           - - files-changed
             - - "mobile/android/config/**"
               - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
               - "**/*.gradle"
 
     android-test:
         description: "Android armv7 unit tests"
-        attributes:
-            build_platform: android-test
-            build_type: opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             kind: test
             tier: 2
             symbol: tc(test)
         worker-type: aws-provisioner-v1/gecko-{level}-b-android
         worker:
             implementation: docker-worker
@@ -110,19 +105,16 @@ jobs:
           - docker-worker:relengapi-proxy:tooltool.download.public
         optimizations:
           - - files-changed
             - - "mobile/android/base/**"
               - "mobile/android/tests/background/junit4/**"
 
     android-lint:
         description: "Android lint"
-        attributes:
-            build_platform: android-lint
-            build_type: opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             kind: test
             tier: 2
             symbol: tc(lint)
         worker-type: aws-provisioner-v1/gecko-{level}-b-android
         worker:
             implementation: docker-worker
@@ -181,19 +173,16 @@ jobs:
               - "mobile/android/**/*.svg"
               - "mobile/android/**/*.xml" # Manifest & android resources
               - "mobile/android/**/*.gradle"
               - "mobile/android/**/Makefile.in"
               - "mobile/android/**/moz.build"
 
     android-checkstyle:
         description: "Android checkstyle"
-        attributes:
-            build_platform: android-checkstyle
-            build_type: opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             kind: test
             tier: 2
             symbol: tc(checkstyle)
         worker-type: aws-provisioner-v1/gecko-{level}-b-android
         worker:
             implementation: docker-worker
@@ -233,19 +222,16 @@ jobs:
             - - "mobile/android/**/checkstyle.xml"
               - "mobile/android/**/*.java"
               - "mobile/android/**/*.gradle"
               - "mobile/android/**/Makefile.in"
               - "mobile/android/**/moz.build"
 
     android-findbugs:
         description: "Android findbugs"
-        attributes:
-            build_platform: android-findbugs
-            build_type: opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             kind: test
             tier: 2
             symbol: tc(findbugs)
         worker-type: aws-provisioner-v1/gecko-{level}-b-android
         worker:
             implementation: docker-worker
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
-   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.try_job:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
    - linux.yml
    - macosx.yml
    - windows.yml
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-linux64-clang/opt:
+linux64-clang:
     description: "Clang toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(clang)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
@@ -17,17 +17,17 @@ linux64-clang/opt:
     run:
         using: toolchain-script
         script: build-clang-linux.sh
         tooltool-downloads: public
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-linux64-clang-tidy/opt:
+linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(clang-tidy)
@@ -41,17 +41,17 @@ linux64-clang-tidy/opt:
         using: toolchain-script
         script: build-clang-tidy-linux.sh
         tooltool-downloads: public
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-linux64-gcc/opt:
+linux64-gcc:
     description: "GCC toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(gcc)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
@@ -59,17 +59,17 @@ linux64-gcc/opt:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-gcc-linux.sh
         resources:
             - 'build/unix/build-gcc/**'
 
-linux64-binutils/opt:
+linux64-binutils:
     description: "Binutils toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
@@ -77,17 +77,17 @@ linux64-binutils/opt:
         docker-image: {in-tree: desktop-build}
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-binutils-linux.sh
         resources:
             - 'build/unix/build-binutils/**'
 
-linux64-cctools-port/opt:
+linux64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(cctools)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
@@ -96,17 +96,17 @@ linux64-cctools-port/opt:
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-cctools-port.sh
         tooltool-downloads: public
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-linux64-hfsplus/opt:
+linux64-hfsplus:
     description: "hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
@@ -116,17 +116,17 @@ linux64-hfsplus/opt:
     run:
         using: toolchain-script
         script: build-hfsplus-linux.sh
         tooltool-downloads: public
         resources:
             - 'build/unix/build-hfsplus/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-linux64-libdmg/opt:
+linux64-libdmg:
     description: "libdmg-hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(libdmg-hfs+)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-macosx64-clang/opt:
+macosx64-clang:
     description: "Clang toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(clang)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
     worker:
@@ -17,17 +17,17 @@ macosx64-clang/opt:
     run:
         using: toolchain-script
         script: build-clang-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-macosx64-clang-tidy/opt:
+macosx64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: macosx64-clang-tidy
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(clang-tidy)
@@ -41,17 +41,17 @@ macosx64-clang-tidy/opt:
         using: toolchain-script
         script: build-clang-tidy-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
 
-macosx64-cctools-port/opt:
+macosx64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(cctools)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
     worker:
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -1,13 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-win32-clang-cl/opt:
+win32-clang-cl:
     description: "Clang-cl toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW32(clang-cl)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
@@ -15,17 +15,17 @@ win32-clang-cl/opt:
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-clang32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
 
-win64-clang-cl/opt:
+win64-clang-cl:
     description: "Clang-cl toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW64(clang-cl)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
@@ -33,17 +33,17 @@ win64-clang-cl/opt:
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-clang64-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper64.sh'
 
-win32-clang-tidy/opt:
+win32-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win32-clang-tidy
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW32(clang-tidy)
@@ -54,17 +54,17 @@ win32-clang-tidy/opt:
         max-run-time: 36000
     run:
         using: toolchain-script
         script: build-clang-tidy32-windows.sh
         resources:
             - 'build/build-clang/**'
             - 'taskcluster/scripts/misc/build-clang-windows-helper32.sh'
 
-win64-clang-tidy/opt:
+win64-clang-tidy:
     description: "Clang-tidy toolchain build"
     index:
         product: static-analysis
         job-name: win64-clang-tidy
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TW64(clang-tidy)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/try_job.py
@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_job_try_name(config, jobs):
+    """
+    For a task which is governed by `-j` in try syntax, set the `job_try_name`
+    attribute based on the job name.
+    """
+    for job in jobs:
+        job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
+        yield job
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -27,22 +27,16 @@ BUILD_KINDS = set([
     'artifact-build',
     'hazard',
     'l10n',
     'valgrind',
     'static-analysis',
     'spidermonkey',
 ])
 
-# anything in this list is governed by -j, matching against the `build_platform` attribute
-JOB_KINDS_MATCHING_BUILD_PLATFORM = set([
-    'toolchain',
-    'android-stuff',
-])
-
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)
 
 
 def alias_contains(infix):
@@ -590,26 +584,19 @@ class TryOptionSyntax(object):
                 return False
             return True
 
         job_try_name = attr('job_try_name')
         if job_try_name:
             if self.jobs is None or job_try_name in self.jobs:
                 if self.platforms is None or attr('build_platform') not in self.platforms:
                     return True
-
-        if attr('kind') == 'test':
+        elif attr('kind') == 'test':
             return match_test(self.unittests, 'unittest_try_name') \
                  or match_test(self.talos, 'talos_try_name')
-        elif attr('kind') in JOB_KINDS_MATCHING_BUILD_PLATFORM:
-            # This will add 'job' tasks to the target set even if no try syntax was specified.
-            if not self.jobs:
-                return True
-            if attr('build_platform') in self.jobs:
-                return True
         elif attr('kind') in BUILD_KINDS:
             if attr('build_type') not in self.build_types:
                 return False
             elif self.platforms is None:
                 # for "-p all", look for try in the 'run_on_projects' attribute
                 return check_run_on_projects()
             else:
                 if attr('build_platform') not in self.platforms: