Bug 1279221 - Remove buildbot routes for Taskcluster tasks; r?dustin draft
authorMike Shal <mshal@mozilla.com>
Tue, 29 Nov 2016 15:02:00 -0500
changeset 446770 c79ad6c48b448a7ca7f0c9d6372a40dd1a54bc2b
parent 445882 741a720c98cdb92c229376be0badbf036f653bff
child 538859 f78984c5a756f8bad8f12f6dc9242f18f036b131
push id37874
push userbmo:mshal@mozilla.com
push dateThu, 01 Dec 2016 20:48:46 +0000
reviewersdustin
bugs1279221
milestone53.0a1
Bug 1279221 - Remove buildbot routes for Taskcluster tasks; r?dustin MozReview-Commit-ID: A2YxsnWo9ek
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/build/windows.yml
taskcluster/ci/hazard/kind.yml
taskcluster/ci/l10n/kind.yml
taskcluster/ci/spidermonkey/kind.yml
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -1,15 +1,13 @@
 android-api-15/debug:
     description: "Android 4.0 API15+ Debug"
     index:
         product: mobile
-        job-name:
-            buildbot: android-api-15-debug
-            gecko-v2: android-api-15-debug
+        job-name: android-api-15-debug
     treeherder:
         platform: android-4-0-armv7-api15/debug
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
     run:
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -21,19 +21,17 @@ linux64/opt:
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
 
 linux64/pgo:
     description: "Linux64 PGO"
     index:
         product: firefox
-        job-name:
-            buildbot: linux64-pgo
-            gecko-v2: linux64-pgo
+        job-name: linux64-pgo
     treeherder:
         platform: linux64/pgo
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
@@ -49,19 +47,17 @@ linux64/pgo:
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
 
 linux64/debug:
     description: "Linux64 Debug"
     index:
         product: firefox
-        job-name:
-            buildbot: linux64-debug
-            gecko-v2: linux64-debug
+        job-name: linux64-debug
     treeherder:
         platform: linux64/debug
         symbol: tc(B)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
     run:
@@ -126,18 +122,17 @@ linux/debug:
         custom-build-variant-cfg: debug
         tooltool-downloads: public
         need-xvfb: true
 
 linux/pgo:
     description: "Linux32 PGO"
     index:
         product: firefox
-        job-name:
-            gecko-v2: linux-pgo
+        job-name: linux-pgo
     treeherder:
         platform: linux32/pgo
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -1,14 +1,13 @@
 win32/debug:
     description: "Win32 Debug"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win32-debug
+        job-name: win32-debug
     treeherder:
         platform: windows2012-32/debug
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 7200
@@ -17,18 +16,17 @@ win32/debug:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_win32_debug.py
 
 win32/opt:
     description: "Win32 Opt"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win32-opt
+        job-name: win32-opt
     treeherder:
         platform: windows2012-32/opt
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 7200
@@ -37,18 +35,17 @@ win32/opt:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_win32_opt.py
 
 win32/pgo:
     description: "Win32 Opt PGO"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win32-pgo
+        job-name: win32-pgo
     treeherder:
         platform: windows2012-32/pgo
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 9000
@@ -58,18 +55,17 @@ win32/pgo:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_win32_opt.py
 
 win64/debug:
     description: "Win64 Debug"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win64-debug
+        job-name: win64-debug
     treeherder:
         platform: windows2012-64/debug
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 7200
@@ -78,18 +74,17 @@ win64/debug:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_win64_debug.py
 
 win64/opt:
     description: "Win64 Opt"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win64-opt
+        job-name: win64-opt
     treeherder:
         platform: windows2012-64/opt
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 7200
@@ -98,18 +93,17 @@ win64/opt:
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_win64_opt.py
 
 win64/pgo:
     description: "Win64 Opt PGO"
     index:
         product: firefox
-        job-name:
-            gecko-v2: win64-pgo
+        job-name: win64-pgo
     treeherder:
         platform: windows2012-64/pgo
         symbol: tc(B)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         implementation: generic-worker
         max-run-time: 10800
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -19,18 +19,17 @@ job-defaults:
         max-run-time: 36000
         docker-image: {in-tree: desktop-build}
 
 jobs:
     linux64-shell-haz/debug:
         description: "JS Shell Hazard Analysis Linux"
         index:
             product: firefox
-            job-name:
-                gecko-v2: shell-haz-debug
+            job-name: shell-haz-debug
         treeherder:
             platform: linux64/debug
             symbol: SM-tc(H)
         run:
             using: hazard
             tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest"
             command: >
                 cd /home/worker/checkouts/gecko/taskcluster/scripts/builder
@@ -39,18 +38,17 @@ jobs:
             files-changed:
                 - js/public/**
                 - js/src/**
 
     linux64-haz/debug:
         description: "Browser Hazard Analysis Linux"
         index:
             product: firefox
-            job-name:
-                gecko-v2: browser-haz-debug
+            job-name: browser-haz-debug
         treeherder:
             platform: linux64/debug
             symbol: tc(H)
         run:
             using: hazard
             tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest"
             mozconfig: "browser/config/mozconfigs/linux64/hazards"
             command: >
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -42,18 +42,17 @@ job-defaults:
             - testing/mozharness/scripts/desktop_l10n.py
             - toolkit/locales/**
             - toolkit/mozapps/installer/**
 
 jobs:
     linux-l10n/opt:
         description: "Localization"
         index:
-            job-name:
-                gecko-v2: linux32-l10n-opt
+            job-name: linux32-l10n-opt
         treeherder:
             platform: linux32/opt
             symbol: tc(L10n)
         run:
             script: mozharness/scripts/desktop_l10n.py
             actions: [clone-locales list-locales setup repack summary]
             config:
                 - single_locale/tc_linux32.py
@@ -64,18 +63,17 @@ jobs:
                 - total-chunks=1
                 - this-chunk=1
             tooltool-downloads: public
             need-xvfb: true
 
     linux64-l10n/opt:
         description: "Localization"
         index:
-            job-name:
-                gecko-v2: linux64-l10n-opt
+            job-name: linux64-l10n-opt
         treeherder:
             platform: linux64/opt
             symbol: tc(L10n)
         run:
             script: mozharness/scripts/desktop_l10n.py
             actions: [clone-locales list-locales setup repack summary]
             config:
                 - single_locale/tc_linux64.py
@@ -87,18 +85,17 @@ jobs:
                 - this-chunk=1
             tooltool-downloads: public
             need-xvfb: true
 
     android-api-15-l10n/opt:
         description: "Single Locale Repack"
         index:
             product: mobile
-            job-name:
-                gecko-v2: android-l10n-opt
+            job-name: android-l10n-opt
         treeherder:
             platform: android-4-0-armv7-api15/opt
             symbol: tc(L10n)
         worker-type: aws-provisioner-v1/gecko-{level}-b-android
         worker:
             max-run-time: 18000
         run:
             script: mozharness/scripts/mobile_l10n.py
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -29,19 +29,17 @@ job-defaults:
             # appended to this list
             - js/public/**
             - js/src/**
 
 jobs:
     sm-package/opt:
         description: "Spidermonkey source package and test"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-package-opt
+            job-name: sm-package-opt
         treeherder:
             symbol: SM-tc(pkg)
         run:
             using: spidermonkey-package
             spidermonkey-variant: plain
         when:
             files-changed:
                 - build/**
@@ -66,34 +64,31 @@ jobs:
                 - testing/mozbase/**
                 - test.mozbuild
                 - toolkit/mozapps/installer/package-name.mk
                 - toolkit/mozapps/installer/upload-files.mk
 
     sm-mozjs-sys/debug:
         description: "Build js/src as the mozjs_sys Rust crate"
         index:
-            job-name:
-                gecko-v2: sm-mozjs-sys-debug
+            job-name: sm-mozjs-sys-debug
         treeherder:
             symbol: SM-tc(mozjs-crate)
         run:
             using: spidermonkey-mozjs-crate
             spidermonkey-variant: plain
         run-on-projects:
             - integration
             - release
             - try
 
     sm-plain/debug:
         description: "Spidermonkey Plain"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-plaindebug-debug
+            job-name: sm-plaindebug-debug
         treeherder:
             platform: linux64/debug
             symbol: SM-tc(p)
         run:
             spidermonkey-variant: plaindebug
 
     sm-plain/opt:
         description: "Spidermonkey Plain"
@@ -102,98 +97,82 @@ jobs:
         treeherder:
             symbol: SM-tc(p)
         run:
             spidermonkey-variant: plain
 
     sm-arm-sim/debug:
         description: "Spidermonkey ARM sim"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-arm-sim-debug
+            job-name: sm-arm-sim-debug
         treeherder:
             symbol: SM-tc(arm)
         run:
             spidermonkey-variant: arm-sim
 
     sm-arm64-sim/debug:
         description: "Spidermonkey ARM64 sim"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-arm64-sim-debug
+            job-name: sm-arm64-sim-debug
         treeherder:
             symbol: SM-tc(arm64)
         run:
             spidermonkey-variant: arm64-sim
 
     sm-asan/opt:
         description: "Spidermonkey Address Sanitizer"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-asan-opt
+            job-name: sm-asan-opt
         treeherder:
             symbol: SM-tc(asan)
         run:
             spidermonkey-variant: asan
             tooltool-manifest: browser/config/tooltool-manifests/linux64/asan.manifest
 
     sm-compacting/debug:
         description: "Spidermonkey Compacting"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-compacting-debug
+            job-name: sm-compacting-debug
         treeherder:
             symbol: SM-tc(cgc)
         run:
             spidermonkey-variant: compacting
 
     sm-msan/opt:
         description: "Spidermonkey Memory Sanitizer"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-msan-opt
+            job-name: sm-msan-opt
         treeherder:
             symbol: SM-tc(msan)
         run:
             spidermonkey-variant: msan
             tooltool-manifest: browser/config/tooltool-manifests/linux64/msan.manifest
 
     sm-tsan/opt:
         description: "Spidermonkey Thread Sanitizer"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-tsan-opt
+            job-name: sm-tsan-opt
         treeherder:
             symbol: SM-tc(tsan)
             tier: 3
         run-on-projects: []
         run:
             spidermonkey-variant: tsan
             tooltool-manifest: browser/config/tooltool-manifests/linux64/tsan.manifest
 
     sm-rootanalysis/debug:
         description: "Spidermonkey Root Analysis"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-rootanalysis-debug
+            job-name: sm-rootanalysis-debug
         treeherder:
             symbol: SM-tc(r)
         run:
             spidermonkey-variant: rootanalysis
 
     sm-nonunified/debug:
         description: "Spidermonkey Non-Unified Debug"
         index:
-            job-name:
-                buildbot: sm-plain
-                gecko-v2: sm-nonunified-debug
+            job-name: sm-nonunified-debug
         treeherder:
             platform: linux64/debug
             symbol: SM-tc(nu)
         run:
             spidermonkey-variant: nonunified
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -67,13 +67,13 @@ JOB_NAME_WHITELIST = set([
     'win32-opt',
     'win32-pgo',
     'win64-debug',
     'win64-opt',
     'win64-pgo',
 ])
 
 JOB_NAME_WHITELIST_ERROR = """\
-The gecko-v2 job name {} is not in the whitelist in __file__.
+The gecko-v2 job name {} is not in the whitelist in gecko_v2_whitelist.py.
 If this job runs on Buildbot, please ensure that the job names match between
 Buildbot and TaskCluster, then add the job name to the whitelist.  If this is a
 new job, there is nothing to check -- just add the job to the whitelist.
 """
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -84,30 +84,17 @@ task_description_schema = Schema({
 
     # information for indexing this build so its artifacts can be discovered;
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any('firefox', 'mobile'),
 
         # the names to use for this job in the TaskCluster index
-        'job-name': Any(
-            # Assuming the job is named "normally", this is the v2 job name,
-            # and the v1 and buildbot routes will be determined appropriately.
-            basestring,
-
-            # otherwise, give separate names for each of the legacy index
-            # routes; if a name is omitted, no corresponding route will be
-            # created.
-            {
-                # the name as it appears in buildbot routes
-                Optional('buildbot'): basestring,
-                Required('gecko-v2'): basestring,
-            }
-        ),
+        'job-name': basestring,
 
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
@@ -309,25 +296,20 @@ GROUP_NAMES = {
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
-BUILDBOT_ROUTE_TEMPLATES = [
-    "index.buildbot.branches.{project}.{job-name-buildbot}",
-    "index.buildbot.revisions.{head_rev}.{project}.{job-name-buildbot}",
-]
-
 V2_ROUTE_TEMPLATES = [
-    "index.gecko.v2.{project}.latest.{product}.{job-name-gecko-v2}",
-    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name-gecko-v2}",
-    "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name-gecko-v2}",
+    "index.gecko.v2.{project}.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
+    "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
@@ -516,40 +498,27 @@ def add_index_routes(config, tasks):
         index = task.get('index')
         routes = task.setdefault('routes', [])
 
         if not index:
             yield task
             continue
 
         job_name = index['job-name']
-        # unpack the v2 name to v1 and buildbot names
-        if isinstance(job_name, basestring):
-            base_name, type_name = job_name.rsplit('-', 1)
-            job_name = {
-                'buildbot': base_name,
-                'gecko-v2': '{}-{}'.format(base_name, type_name),
-            }
-
-        if job_name['gecko-v2'] not in JOB_NAME_WHITELIST:
-            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name['gecko-v2']))
+        if job_name not in JOB_NAME_WHITELIST:
+            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
 
         subs = config.params.copy()
-        for n in job_name:
-            subs['job-name-' + n] = job_name[n]
+        subs['job-name'] = job_name
         subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
                                                 time.gmtime(config.params['build_date']))
         subs['product'] = index['product']
 
-        if 'buildbot' in job_name:
-            for tpl in BUILDBOT_ROUTE_TEMPLATES:
-                routes.append(tpl.format(**subs))
-        if 'gecko-v2' in job_name:
-            for tpl in V2_ROUTE_TEMPLATES:
-                routes.append(tpl.format(**subs))
+        for tpl in V2_ROUTE_TEMPLATES:
+            routes.append(tpl.format(**subs))
 
         # The default behavior is to rank tasks according to their tier
         extra_index = task.setdefault('extra', {}).setdefault('index', {})
         rank = index.get('rank', 'by-tier')
 
         if rank == 'by-tier':
             # rank is zero for non-tier-1 tasks and based on pushid for others;
             # this sorts tier-{2,3} builds below tier-1 in the index
@@ -660,17 +629,17 @@ def check_v2_routes():
 
     # we only deal with the 'routes' key here
     routes = routes_json['routes']
 
     # we use different variables than mozharness
     for mh, tg in [
             ('{index}', 'index'),
             ('{build_product}', '{product}'),
-            ('{build_name}-{build_type}', '{job-name-gecko-v2}'),
+            ('{build_name}-{build_type}', '{job-name}'),
             ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
         routes = [r.replace(mh, tg) for r in routes]
 
     if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
         raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
                         "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
 
 check_v2_routes()