Bug 1280129 - Update tasks to use v2 treeherder routes r=dustin
authorGregory Arndt <garndt@mozilla.com>
Tue, 14 Jun 2016 12:56:25 -0500
changeset 378248 099196831d4cf158176631064079088d84abf8db
parent 378247 b850837edec241b11e9922b382d9178b265e55b8
child 378266 f3d5b3e79015645ae443a9841c8914cd9569ff61
push id20971
push usergarndt@mozilla.com
push dateWed, 15 Jun 2016 14:35:03 +0000
reviewersdustin
bugs1280129
milestone50.0a1
Bug 1280129 - Update tasks to use v2 treeherder routes r=dustin Jobs reporting to treeherder should rely on the task route for project, revision, and pushlog ID rather than things stuffed into task.extra.treeherder. This also removes the need for a revision_hash that was calculated by mozilla-taskcluster. MozReview-Commit-ID: EcQM9QRZzgG
.taskcluster.yml
taskcluster/ci/docker-image/image.yml
taskcluster/ci/legacy/tasks/decision/branch.yml
taskcluster/ci/legacy/tasks/decision/try.yml
taskcluster/docs/parameters.rst
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/kind/docker_image.py
taskcluster/taskgraph/kind/legacy.py
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,18 +55,18 @@ tasks:
         - "docker-worker:cache:level-{{level}}-*"
         - "docker-worker:cache:tooltool-cache"
         - "secrets:get:project/taskcluster/gecko/hgfingerprint"
         # mozilla-taskcluster will append the appropriate assume:repo:<repo>
         # scope here.
 
       routes:
         - "index.gecko.v2.{{project}}.latest.firefox.decision"
-        - "tc-treeherder.{{project}}.{{revision_hash}}"
-        - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+        - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
+        - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}"
 
       payload:
         env:
           # checkout-gecko uses these to check out the source; the inputs
           # to `mach taskgraph decision` are all on the command line.
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
--- a/taskcluster/ci/docker-image/image.yml
+++ b/taskcluster/ci/docker-image/image.yml
@@ -13,18 +13,18 @@ task:
   workerType: taskcluster-images
   provisionerId: aws-provisioner-v1
   schedulerId: task-graph-scheduler
 
   routes:
       - index.docker.images.v1.{{project}}.{{image_name}}.latest
       - index.docker.images.v1.{{project}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}}
       - index.docker.images.v1.{{project}}.{{image_name}}.hash.{{context_hash}}
-      - tc-treeherder.{{project}}.{{revision_hash}}
-      - tc-treeherder-stage.{{project}}.{{revision_hash}}
+      - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+      - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
 
   payload:
     env:
       HASH: '{{context_hash}}'
       PROJECT: '{{project}}'
       CONTEXT_URL: '{{context_url}}'
       CONTEXT_PATH: '{{context_path}}'
       BASE_REPOSITORY: '{{base_repository}}'
@@ -45,14 +45,12 @@ task:
         path: '/artifacts/image.tar'
         expires: '{{#from_now}}1 year{{/from_now}}'
   extra:
     treeherderEnv:
       - staging
       - production
     treeherder:
       jobKind: other
-      revision: {{head_rev}}
-      revision_hash: {{revision_hash}}
       build:
         platform: 'taskcluster-images'
       symbol: 'I'
 
--- a/taskcluster/ci/legacy/tasks/decision/branch.yml
+++ b/taskcluster/ci/legacy/tasks/decision/branch.yml
@@ -12,18 +12,18 @@ scopes:
   - queue:*
   - docker-worker:*
   - scheduler:*
 tasks:
   - taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
     task:
       routes:
         - index.gecko.v2.{{project}}.latest.firefox.decision
-        - tc-treeherder-stage.{{project}}.{{revision_hash}}
-        - tc-treeherder.{{project}}.{{revision_hash}}
+        - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
+        - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
       created: '{{now}}'
       deadline: '{{#from_now}}1 day{{/from_now}}'
       metadata:
         owner: mozilla-taskcluster-maintenance@mozilla.com
         source: {{{source}}}
         name: Initial decision task for {{project}} specific graphs
         description: |
           This is the single most important task as it decides how all other tasks
@@ -74,26 +74,23 @@ tasks:
             checkout-gecko workspace &&
             cd workspace/gecko &&
             ./mach taskcluster-graph
             --pushlog-id='{{pushlog_id}}'
             --project='{{project}}'
             --message='{{comment}}'
             --owner='{{owner}}'
             --level='{{level}}'
-            --revision-hash='{{revision_hash}}'
             --extend-graph > /home/worker/artifacts/graph.json &&
             cp /home/worker/artifacts/graph.json /home/worker/artifacts/all_tasks.json
         graphs:
           - /home/worker/artifacts/graph.json
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             # Arbitrary value for keeping these artifacts around.  They are just the
             # graph.json and context directories for now, so nothing that needs
             # to stay around for long.
             expires: '{{#from_now}}7 days{{/from_now}}'
       extra:
         treeherder:
           symbol: D
-          revision: '{{revision}}'
-          revision_hash: '{{revision_hash}}'
--- a/taskcluster/ci/legacy/tasks/decision/try.yml
+++ b/taskcluster/ci/legacy/tasks/decision/try.yml
@@ -36,18 +36,18 @@ tasks:
         # Bug 1269443: cache scopes, etc. must be listed explicitly
         - "docker-worker:cache:level-1-*"
         - "docker-worker:cache:tooltool-cache"
         - "secrets:get:project/taskcluster/gecko/hgfingerprint"
         - "assume:repo:hg.mozilla.org/try:*"
 
       routes:
         - "index.gecko.v2.{{project}}.latest.firefox.decision"
-        - "tc-treeherder.{{project}}.{{revision_hash}}"
-        - "tc-treeherder-stage.{{project}}.{{revision_hash}}"
+        - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
+        - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
 
       payload:
         env:
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
           GECKO_HEAD_REPOSITORY: '{{{url}}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
 
@@ -85,21 +85,18 @@ tasks:
             --project='{{project}}'
             --message='{{comment}}'
             --owner='{{owner}}'
             --level='{{level}}'
             --base-repository='https://hg.mozilla.org/mozilla-central'
             --head-repository='{{{url}}}'
             --head-ref='{{revision}}'
             --head-rev='{{revision}}'
-            --revision-hash='{{revision_hash}}'
 
         artifacts:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
             expires: '{{#from_now}}7 days{{/from_now}}'
 
       extra:
         treeherder:
           symbol: D
-          revision: '{{revision}}'
-          revision_hash: '{{revision_hash}}'
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -30,19 +30,16 @@ Push Information
    The revision to check out; this can be a short revision string
 
 ``head_ref``
    For Mercurial repositories, this is the same as ``head_rev``.  For
    git repositories, which do not allow pulling explicit revisions, this gives
    the symbolic ref containing ``head_rev`` that should be pulled from
    ``head_repository``.
 
-``revision_hash``
-   The full-length revision string
-
 ``owner``
    Email address indicating the person who made the push.  Note that this
    value may be forged and *must not* be relied on for authentication.
 
 ``message``
    The commit message
 
 ``pushlog_id``
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -85,17 +85,16 @@ def get_decision_parameters(options):
     This also applies per-project parameters, based on the given project.
 
     """
     parameters = {n: options[n] for n in [
         'base_repository',
         'head_repository',
         'head_rev',
         'head_ref',
-        'revision_hash',
         'message',
         'project',
         'pushlog_id',
         'owner',
         'level',
         'target_tasks_method',
     ] if n in options}
 
--- a/taskcluster/taskgraph/kind/docker_image.py
+++ b/taskcluster/taskgraph/kind/docker_image.py
@@ -32,32 +32,32 @@ INDEX_URL = 'https://index.taskcluster.n
 
 class DockerImageKind(base.Kind):
 
     def load_tasks(self, params):
         # TODO: make this match the pushdate (get it from a parameter rather than vcs)
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
 
         parameters = {
+            'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': pushdate,
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
             'month': pushdate[4:6],
             'day': pushdate[6:8],
             'project': params['project'],
             'docker_image': docker_image,
             'base_repository': params['base_repository'] or params['head_repository'],
             'head_repository': params['head_repository'],
             'head_ref': params['head_ref'] or params['head_rev'],
             'head_rev': params['head_rev'],
             'owner': params['owner'],
             'level': params['level'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
-            'revision_hash': params['revision_hash'],
             'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
                     .format(repo=params['head_repository'], rev=params['head_rev']),
         }
 
         tasks = []
         templates = Templates(self.path)
         for image_name in self.config['images']:
             context_path = os.path.join('testing', 'docker', image_name)
--- a/taskcluster/taskgraph/kind/legacy.py
+++ b/taskcluster/taskgraph/kind/legacy.py
@@ -34,17 +34,16 @@ GECKO = os.path.realpath(os.path.join(RO
 TASKID_PLACEHOLDER = 'TaskLabel=={}'
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 DEFAULT_TRY = 'try: -b do -p all -u all -t all'
 DEFAULT_JOB_PATH = os.path.join(
     'tasks', 'branches', 'base_jobs.yml'
 )
 
-TREEHERDER_ROUTE_PREFIX = 'tc-treeherder-stage'
 TREEHERDER_ROUTES = {
     'staging': 'tc-treeherder-stage',
     'production': 'tc-treeherder'
 }
 
 # time after which a try build's results will expire
 TRY_EXPIRATION = "14 days"
 
@@ -205,45 +204,48 @@ def set_expiration(task, timestamp):
         return
 
     # for docker-worker, artifacts is a dictionary
     # for generic-worker, artifacts is a list
     # for taskcluster-worker, it will depend on what we do in artifacts plugin
     for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
         artifact['expires'] = timestamp
 
-def add_treeherder_revision_info(task, revision, revision_hash):
-    # Only add treeherder information if task.extra.treeherder is present
-    if 'extra' not in task and 'treeherder' not in task.extra:
-        return
+def format_treeherder_route(destination, project, revision, pushlog_id):
+    return "{}.v2.{}.{}.{}".format(destination,
+                                   project,
+                                   revision,
+                                   pushlog_id)
 
-    task['extra']['treeherder']['revision'] = revision
-    task['extra']['treeherder']['revision_hash'] = revision_hash
-
-
-def decorate_task_treeherder_routes(task, suffix):
+def decorate_task_treeherder_routes(task, project, revision, pushlog_id):
     """Decorate the given task with treeherder routes.
 
     Uses task.extra.treeherderEnv if available otherwise defaults to only
     staging.
 
     :param dict task: task definition.
-    :param str suffix: The project/revision_hash portion of the route.
+    :param str project: The project the tasks are running for.
+    :param str revision: The revision for the push
+    :param str pushlog_id: The ID of the push
     """
 
     if 'extra' not in task:
         return
 
     if 'routes' not in task:
         task['routes'] = []
 
     treeheder_env = task['extra'].get('treeherderEnv', ['staging'])
 
     for env in treeheder_env:
-        task['routes'].append('{}.{}'.format(TREEHERDER_ROUTES[env], suffix))
+        route = format_treeherder_route(TREEHERDER_ROUTES[env],
+                                        project,
+                                        revision,
+                                        pushlog_id)
+        task['routes'].append(route)
 
 def decorate_task_json_routes(task, json_routes, parameters):
     """Decorate the given task with routes.json routes.
 
     :param dict task: task definition.
     :param json_routes: the list of routes to use from routes.json
     :param parameters: dictionary of parameters to use in route templates
     """
@@ -337,42 +339,36 @@ class LegacyKind(base.Kind):
             'pushtime': pushdate[8:],
             'year': pushdate[0:4],
             'month': pushdate[4:6],
             'day': pushdate[6:8],
             'owner': params['owner'],
             'level': params['level'],
             'from_now': json_time_from_now,
             'now': current_json_time(),
-            'revision_hash': params['revision_hash']
         }.items())
 
-        treeherder_route = '{}.{}'.format(
-            params['project'],
-            params.get('revision_hash', '')
-        )
-
         routes_file = os.path.join(root, 'routes.json')
         with open(routes_file) as f:
             contents = json.load(f)
             json_routes = contents['routes']
             # TODO: Nightly and/or l10n routes
 
         # Task graph we are generating for taskcluster...
         graph = {
             'tasks': [],
             'scopes': set(),
         }
 
-        if params['revision_hash']:
-            for env in TREEHERDER_ROUTES:
-                route = 'queue:route:{}.{}'.format(
-                    TREEHERDER_ROUTES[env],
-                    treeherder_route)
-                graph['scopes'].add(route)
+        for env in TREEHERDER_ROUTES:
+            route = format_treeherder_route(TREEHERDER_ROUTES[env],
+                                            parameters['project'],
+                                            parameters['head_rev'],
+                                            parameters['pushlog_id'])
+            graph['scopes'].add("queue:route:{}".format(route))
 
         graph['metadata'] = {
             'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
             'description': 'Task graph generated via ./mach taskcluster-graph',
             'name': 'task graph local'
         }
@@ -433,25 +429,23 @@ class LegacyKind(base.Kind):
 
             set_interactive_task(build_task, interactive)
 
             # try builds don't use cache
             if project == "try":
                 remove_caches_from_task(build_task)
                 set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
 
-            if params['revision_hash']:
-                add_treeherder_revision_info(build_task['task'],
-                                             params['head_rev'],
-                                             params['revision_hash'])
-                decorate_task_treeherder_routes(build_task['task'],
-                                                treeherder_route)
-                decorate_task_json_routes(build_task['task'],
-                                          json_routes,
-                                          build_parameters)
+            decorate_task_treeherder_routes(build_task['task'],
+                                            build_parameters['project'],
+                                            build_parameters['head_rev'],
+                                            build_parameters['pushlog_id'])
+            decorate_task_json_routes(build_task['task'],
+                                      json_routes,
+                                      build_parameters)
 
             # Ensure each build graph is valid after construction.
             validate_build_task(build_task)
             attributes = build_task['attributes'] = {'kind':'legacy', 'legacy_kind': 'build'}
             if 'build_name' in build:
                 attributes['build_platform'] = build['build_name']
             if 'build_type' in task_extra:
                 attributes['build_type'] = {'dbg': 'debug'}.get(task_extra['build_type'],
@@ -510,19 +504,16 @@ class LegacyKind(base.Kind):
                 post_parameters = merge_dicts(build_parameters,
                                               post_build.get('additional-parameters', {}))
                 post_task = configure_dependent_task(post_build['task'],
                                                      post_parameters,
                                                      mklabel(),
                                                      templates,
                                                      build_treeherder_config)
                 set_interactive_task(post_task, interactive)
-                add_treeherder_revision_info(post_task['task'],
-                                             params['head_rev'],
-                                             params['revision_hash'])
 
                 if project == "try":
                     set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
 
                 post_task['attributes'] = attributes.copy()
                 post_task['attributes']['legacy_kind'] = 'post_build'
                 post_task['attributes']['post_build'] = post_build['job_flag']
                 graph['tasks'].append(post_task)
@@ -560,24 +551,20 @@ class LegacyKind(base.Kind):
                         test_parameters['chunk'] = chunk
                     test_task = configure_dependent_task(test['task'],
                                                          test_parameters,
                                                          mklabel(),
                                                          templates,
                                                          build_treeherder_config)
                     set_interactive_task(test_task, interactive)
 
-                    if params['revision_hash']:
-                        add_treeherder_revision_info(test_task['task'],
-                                                     params['head_rev'],
-                                                     params['revision_hash'])
-                        decorate_task_treeherder_routes(
-                            test_task['task'],
-                            treeherder_route
-                        )
+                    decorate_task_treeherder_routes(test_task['task'],
+                                                    test_parameters['project'],
+                                                    test_parameters['head_rev'],
+                                                    test_parameters['pushlog_id'])
 
                     if project == "try":
                         set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
 
                     test_task['attributes'] = attributes.copy()
                     test_task['attributes']['legacy_kind'] = 'unittest'
                     test_task['attributes']['test_platform'] = attributes['build_platform']
                     test_task['attributes']['unittest_try_name'] = test['unittest_try_name']