Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin draft
authorJustin Wood <Callek@gmail.com>
Mon, 09 Jan 2017 16:23:04 -0500
changeset 462617 b3254f6cc3f6f526ec3877add890d12a8947080c
parent 462402 3e275d37a06236981bff399b7d7aa0646be3fee7
child 542459 79516e258ef7ef168da1db1fffb8658bf7481837
push id41825
push userCallek@gmail.com
push dateTue, 17 Jan 2017 20:47:26 +0000
reviewersdustin
bugs1301495
milestone53.0a1
Bug 1301495 - Taskcluster l10n indexing should match mozharness' l10n indexing. r=dustin Adds l10n and nightly indexing, matching (better) what Buildbot is currently doing with these types of tasks (This patch is against `date`, will be grafted on review for real landing, using autoland) MozReview-Commit-ID: K0BYwaCm6xL
taskcluster/ci/build/android.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/l10n/kind.yml
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -48,17 +48,18 @@ android-x86/opt:
         tooltool-downloads: internal
 
 android-x86-nightly/opt:
     description: "Android 4.2 x86 Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-x86-nightly-opt
+        job-name: android-x86-opt
+        type: nightly
     treeherder:
         platform: android-4-2-x86/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
@@ -102,17 +103,18 @@ android-api-15/opt:
         tooltool-downloads: internal
 
 android-api-15-nightly/opt:
     description: "Android 4.0 API15+ Nightly"
     attributes:
         nightly: true
     index:
         product: mobile
-        job-name: android-api-15-nightly-opt
+        job-name: android-api-15-opt
+        type: nightly
     treeherder:
         platform: android-4-0-armv7-api15/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-android
     worker:
         implementation: docker-worker
         max-run-time: 7200
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -150,17 +150,18 @@ linux/pgo:
         need-xvfb: true
 
 linux-nightly/opt:
     description: "Linux32 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux32-nightly-opt
+        job-name: linux-opt
+        type: nightly
     treeherder:
         platform: linux32/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
@@ -227,17 +228,18 @@ linux64-asan/debug:
         need-xvfb: true
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
     index:
         product: firefox
-        job-name: linux64-nightly-opt
+        job-name: linux64-opt
+        type: nightly
     treeherder:
         platform: linux64/opt
         symbol: tc(N)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -39,8 +39,10 @@ macosx64/opt:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
             - builds/releng_base_mac_64_cross_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
+
+        type: nightly
\ No newline at end of file
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@@ -32,26 +32,27 @@ job-template:
       by-build-platform:
          default: 36000
          android-api-15-l10n: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-l10n: internal
    index:
+      type: l10n
       product:
          by-build-platform:
             default: firefox
             android-api-15-l10n: mobile
       job-name:
          by-build-platform:
-            linux-l10n: linux32-l10n-opt
-            linux64-l10n: linux64-l10n-opt
-            macosx64: macosx64-l10n-opt
-            android-api-15-l10n: android-l10n-opt
+            linux-l10n: linux-opt
+            linux64-l10n: linux64-opt
+            macosx64: macosx64-opt
+            android-api-15-l10n: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc(L10n)
       tier:
          by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -31,27 +31,28 @@ job-template:
    run-time:
       by-build-platform:
          default: 36000
          android-api-15-nightly: 18000
    tooltool:
       by-build-platform:
          default: public
          android-api-15-nightly: internal
-#   index:
-#      product:
-#         by-build-platform:
-#            default: firefox
-#            android-api-15-nightly: mobile
-#      job-name:
-#         by-build-platform:
-#            linux-nightly: linux32-nightly-l10n-opt
-#            linux64-nightly: linux64-nightly-l10n-opt
-#            macosx64-nightly: macosx64-nightly-l10n-opt
-#            android-api-15-nightly: android-nightly-l10n-opt
+   index:
+      type: l10n
+      product:
+         by-build-platform:
+            default: firefox
+            android-api-15-nightly: mobile
+      job-name:
+         by-build-platform:
+            linux-nightly: linux-opt
+            linux64-nightly: linux64-opt
+            macosx64-nightly: macosx64-opt
+            android-api-15-nightly: android-api-15-opt
    worker-type:
       by-build-platform:
          default: aws-provisioner-v1/gecko-{level}-b-linux
          android-api-15-nightly: aws-provisioner-v1/gecko-{level}-b-android
    treeherder:
       symbol: tc-L10n(N)
       tier:
          by-build-platform:
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -63,16 +63,19 @@ l10n_description_schema = Schema({
     },
     # Items for the taskcluster index
     Optional('index'): {
         # Product to identify as in the taskcluster index
         Required('product'): _by_platform(basestring),
 
         # Job name to identify as in the taskcluster index
         Required('job-name'): _by_platform(basestring),
+
+        # Type of index
+        Optional('type'): basestring,
     },
     # Description of the localized task
     Required('description'): _by_platform(basestring),
 
     # task object of the dependent task
     Required('dependent-task'): object,
 
     # worker-type to utilize
@@ -350,16 +353,17 @@ def make_job_description(config, jobs):
             },
             'run-on-projects': [],
         }
 
         if job.get('index'):
             job_description['index'] = {
                 'product': job['index']['product'],
                 'job-name': job['index']['job-name'],
+                'type': job['index'].get('type', 'generic'),
             }
 
         if job.get('dependencies'):
             job_description['dependencies'] = job['dependencies']
         if job.get('env'):
             job_description['worker']['env'] = job['env']
         if job.get('when', {}).get('files-changed'):
             job_description.setdefault('when', {})
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -87,16 +87,19 @@ task_description_schema = Schema({
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
         'product': Any('firefox', 'mobile', 'static-analysis'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
+        # Type of gecko v2 index to use
+        'type': Any('generic', 'nightly', 'l10n'),
+
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
             # builds below tier-1 in the index.
@@ -326,16 +329,29 @@ GROUP_NAMES = {
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
 
+V2_NIGHTLY_TEMPLATES = [
+    "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
+    "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
+]
+
+V2_L10N_TEMPLATES = [
+    "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
+]
+
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
     'production': 'tc-treeherder',
     'staging': 'tc-treeherder-stage',
 }
 
 COALESCE_KEY = 'builds.{project}.{name}'
 
@@ -344,16 +360,26 @@ payload_builders = {}
 
 
 def payload_builder(name):
     def wrap(func):
         payload_builders[name] = func
         return func
     return wrap
 
+# define a collection of index builders, depending on the type implementation
+index_builders = {}
+
+
+def index_builder(name):
+    def wrap(func):
+        index_builders[name] = func
+        return func
+    return wrap
+
 
 @payload_builder('docker-worker')
 def build_docker_worker_payload(config, task, task_def):
     worker = task['worker']
 
     image = worker['docker-image']
     if isinstance(image, dict):
         docker_image_task = 'build-docker-image-' + image['in-tree']
@@ -523,38 +549,104 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             task_description_schema, task,
             "In task {!r}:".format(task.get('label', '?no-label?')))
 
 
+@index_builder('generic')
+def add_generic_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_ROUTE_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('nightly')
+def add_nightly_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['build_date'] = time.strftime("%Y.%m.%d",
+                                       time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    for tpl in V2_NIGHTLY_TEMPLATES:
+        routes.append(tpl.format(**subs))
+
+    return task
+
+
+@index_builder('l10n')
+def add_l10n_index_routes(config, task):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    job_name = index['job-name']
+    if job_name not in JOB_NAME_WHITELIST:
+        raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
+
+    subs = config.params.copy()
+    subs['job-name'] = job_name
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    locales = task['attributes'].get('chunk_locales',
+                                     task['attributes'].get('all_locales'))
+
+    if not locales:
+        raise Exception("Error: Unable to use l10n index for tasks without locales")
+
+    # If there are too many locales, we can't write a route for all of them
+    # See Bug 1323792
+    if len(locales) > 18:  # 18 * 3 = 54, max routes = 64
+        return task
+
+    for locale in locales:
+        for tpl in V2_L10N_TEMPLATES:
+            routes.append(tpl.format(locale=locale, **subs))
+
+    return task
+
+
 @transforms.add
 def add_index_routes(config, tasks):
     for task in tasks:
         index = task.get('index')
-        routes = task.setdefault('routes', [])
 
         if not index:
             yield task
             continue
 
-        job_name = index['job-name']
-        if job_name not in JOB_NAME_WHITELIST:
-            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name))
-
-        subs = config.params.copy()
-        subs['job-name'] = job_name
-        subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
-                                                time.gmtime(config.params['build_date']))
-        subs['product'] = index['product']
-
-        for tpl in V2_ROUTE_TEMPLATES:
-            routes.append(tpl.format(**subs))
+        index_type = index.get('type', 'generic')
+        task = index_builders[index_type](config, task)
 
         # The default behavior is to rank tasks according to their tier
         extra_index = task.setdefault('extra', {}).setdefault('index', {})
         rank = index.get('rank', 'by-tier')
 
         if rank == 'by-tier':
             # rank is zero for non-tier-1 tasks and based on pushid for others;
             # this sorts tier-{2,3} builds below tier-1 in the index
@@ -658,24 +750,33 @@ def build_task(config, tasks):
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open("testing/mozharness/configs/routes.json", "rb") as f:
         routes_json = json.load(f)
 
-    # we only deal with the 'routes' key here
-    routes = routes_json['routes']
+    for key in ('routes', 'nightly', 'l10n'):
+        if key == 'routes':
+            tc_template = V2_ROUTE_TEMPLATES
+        elif key == 'nightly':
+            tc_template = V2_NIGHTLY_TEMPLATES
+        elif key == 'l10n':
+            tc_template = V2_L10N_TEMPLATES
+
+        routes = routes_json[key]
 
-    # we use different variables than mozharness
-    for mh, tg in [
-            ('{index}', 'index'),
-            ('{build_product}', '{product}'),
-            ('{build_name}-{build_type}', '{job-name}'),
-            ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
-        routes = [r.replace(mh, tg) for r in routes]
+        # we use different variables than mozharness
+        for mh, tg in [
+                ('{index}', 'index'),
+                ('{build_product}', '{product}'),
+                ('{build_name}-{build_type}', '{job-name}'),
+                ('{year}.{month}.{day}.{pushdate}', '{build_date_long}'),
+                ('{year}.{month}.{day}', '{build_date}')]:
+            routes = [r.replace(mh, tg) for r in routes]
 
-    if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
-        raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
-                        "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
+        if sorted(routes) != sorted(tc_template):
+            raise Exception("V2 TEMPLATES do not match Mozharness's routes.json: "
+                            "(tc):%s vs (mh):%s" % (tc_template, routes))
+
 
 check_v2_routes()