Bug 1380885 - Index more tasks by pushlog id; r?dustin draft
authorGregory Szorc <gps@mozilla.com>
Thu, 13 Jul 2017 16:51:16 -0700
changeset 608659 91bb627beeda4dde974840176a4bf11436786e7f
parent 608658 67cd1ee26f2661fa5efe3d952485ab3c89af4271
child 637384 3e70f38ca25ac1aecaa6de3c6e7baf9aa3af02d5
push id68365
push userbmo:gps@mozilla.com
push dateThu, 13 Jul 2017 23:51:31 +0000
reviewersdustin
bugs1380885, 1380391
milestone56.0a1
Bug 1380885 - Index more tasks by pushlog id; r?dustin 365731510976 (bug 1380391) added index routes for decision tasks by pushlog id. This is a good idea. The pushlog id is guaranteed to always be incrementing (except for repos that are periodically reset, which we don't care about). It is useful to provide strict ordering for pushes and is simpler for machines to consume and sort than dates. So let's index all tasks by pushlog id. MozReview-Commit-ID: BPqx4ARza1c
taskcluster/taskgraph/transforms/task.py
testing/mozharness/configs/routes.json
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -467,16 +467,17 @@ GROUP_NAMES = {
     'SM-tc': 'Spidermonkey builds',
     'pub': 'APK publishing',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
+    "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
 V2_NIGHTLY_TEMPLATES = [
     "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
@@ -1061,16 +1062,17 @@ def check_v2_routes():
         routes = routes_json[key]
 
         # we use different variables than mozharness
         for mh, tg in [
                 ('{index}', 'index'),
                 ('{build_product}', '{product}'),
                 ('{build_name}-{build_type}', '{job-name}'),
                 ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
+                ('{pushid}', '{pushlog_id}'),
                 ('{year}.{month}.{day}', '{build_date}')]:
             routes = [r.replace(mh, tg) for r in routes]
 
         if sorted(routes) != sorted(tc_template):
             raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
                             "(tc):%s vs (mh):%s" % (tc_template, routes))
 
 
--- a/testing/mozharness/configs/routes.json
+++ b/testing/mozharness/configs/routes.json
@@ -1,12 +1,13 @@
 {
     "routes": [
         "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
+        "{index}.gecko.v2.{project}.pushlog-id.{pushid}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
     ],
     "nightly": [
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
     ],
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1382,16 +1382,17 @@ or run without that action (ie: --no-{ac
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
 
         index = self.config.get('taskcluster_index', 'index.garbage.staging')
         fmt = {
             'index': index,
             'project': self.buildbot_config['properties']['branch'],
             'head_rev': revision,
             'pushdate': pushdate,
+            'pushid': pushinfo.pushid,
             'year': pushdate[0:4],
             'month': pushdate[4:6],
             'day': pushdate[6:8],
             'build_product': self.config['stage_product'],
             'build_name': self.query_build_name(),
             'build_type': self.query_build_type(),
             'locale': locale,
         }