Bug 1419561 - Set better nightly l10n routes. r=rail draft
authorJustin Wood <Callek@gmail.com>
Thu, 12 Jan 2017 11:17:32 -0500
changeset 702149 4153bfea52c9414d5eef82d9bbdd23a194bab76b
parent 702148 571fa7413320e0b7ed2cbc2ff1e7d81a1924bd3c
child 702150 083efc49363bbcee7b6d96fe538c74d043a49b2b
push id90398
push userCallek@gmail.com
push dateWed, 22 Nov 2017 20:34:21 +0000
reviewersrail
bugs1419561
milestone59.0a1
Bug 1419561 - Set better nightly l10n routes. r=rail MozReview-Commit-ID: CLUMErDkhVi
taskcluster/ci/nightly-l10n/kind.yml
taskcluster/taskgraph/transforms/task.py
testing/mozharness/configs/routes.json
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@@ -63,17 +63,17 @@ job-template:
    tooltool:
       by-build-platform:
          default: public
          android-api-16-nightly: internal
          macosx64-nightly: internal
          win32-nightly: internal
          win64-nightly: internal
    index:
-      type: l10n
+      type: nightly-l10n
       product:
          by-build-platform:
             default: firefox
             android-api-16-nightly: mobile
       job-name:
          by-build-platform:
             linux-nightly: linux-opt
             linux64-nightly: linux64-opt
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -138,17 +138,18 @@ task_description_schema = Schema({
             'devedition',
             'source',
         ),
 
         # the names to use for this job in the TaskCluster index
         'job-name': basestring,
 
         # Type of gecko v2 index to use
-        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n', 'release'),
+        'type': Any('generic', 'nightly', 'l10n', 'nightly-with-multi-l10n',
+                    'release', 'nightly-l10n'),
 
         # The rank that the task will receive in the TaskCluster
         # index.  A newly completed task supercedes the currently
         # indexed task iff it has a higher rank.  If unspecified,
         # 'by-tier' behavior will be used.
         'rank': Any(
             # Rank is equal the timestamp of the build_date for tier-1
             # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
@@ -578,16 +579,23 @@ V2_TRUNK_ROUTE_TEMPLATES = [
 
 V2_NIGHTLY_TEMPLATES = [
     "index.gecko.v2.{project}.nightly.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}.{job-name}",
 ]
 
+V2_NIGHTLY_L10N_TEMPLATES = [
+    "index.gecko.v2.{project}.nightly.latest.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.nightly.{build_date}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",  # noqa - too long
+    "index.gecko.v2.{project}.nightly.{build_date}.latest.{product}-l10n.{job-name}.{locale}",
+    "index.gecko.v2.{project}.nightly.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
+]
+
 V2_L10N_TEMPLATES = [
     "index.gecko.v2.{project}.revision.{head_rev}.{product}-l10n.{job-name}.{locale}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}-l10n.{job-name}.{locale}",
     "index.gecko.v2.{project}.latest.{product}-l10n.{job-name}.{locale}",
 ]
 
 # the roots of the treeherder routes, keyed by treeherder environment
 TREEHERDER_ROUTE_ROOTS = {
@@ -1189,16 +1197,51 @@ def add_l10n_index_routes(config, task, 
 
     for locale in locales:
         for tpl in V2_L10N_TEMPLATES:
             routes.append(tpl.format(locale=locale, **subs))
 
     return task
 
 
+@index_builder('nightly-l10n')
+def add_nightly_l10n_index_routes(config, task, force_locale=None):
+    index = task.get('index')
+    routes = task.setdefault('routes', [])
+
+    verify_index_job_name(index)
+
+    subs = config.params.copy()
+    subs['job-name'] = index['job-name']
+    subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                            time.gmtime(config.params['build_date']))
+    subs['product'] = index['product']
+
+    locales = task['attributes'].get('chunk_locales',
+                                     task['attributes'].get('all_locales'))
+    # Some tasks has only one locale set
+    if task['attributes'].get('locale'):
+        locales = [task['attributes']['locale']]
+
+    if force_locale:
+        # Used for en-US and multi-locale
+        locales = [force_locale]
+
+    if not locales:
+        raise Exception("Error: Unable to use l10n index for tasks without locales")
+
+    for locale in locales:
+        for tpl in V2_NIGHTLY_L10N_TEMPLATES:
+            routes.append(tpl.format(locale=locale, **subs))
+
+    # Add locales at old route too
+    task = add_l10n_index_routes(config, task, force_locale=force_locale)
+    return task
+
+
 @transforms.add
 def add_index_routes(config, tasks):
     for task in tasks:
         index = task.get('index')
 
         if not index:
             yield task
             continue
@@ -1498,23 +1541,25 @@ def check_run_task_caches(config, tasks)
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open(os.path.join(GECKO, "testing/mozharness/configs/routes.json"), "rb") as f:
         routes_json = json.load(f)
 
-    for key in ('routes', 'nightly', 'l10n'):
+    for key in ('routes', 'nightly', 'l10n', 'nightly-l10n'):
         if key == 'routes':
             tc_template = V2_ROUTE_TEMPLATES
         elif key == 'nightly':
             tc_template = V2_NIGHTLY_TEMPLATES
         elif key == 'l10n':
             tc_template = V2_L10N_TEMPLATES
+        elif key == 'nightly-l10n':
+            tc_template = V2_NIGHTLY_L10N_TEMPLATES + V2_L10N_TEMPLATES
 
         routes = routes_json[key]
 
         # we use different variables than mozharness
         for mh, tg in [
                 ('{index}', 'index'),
                 ('{build_product}', '{product}'),
                 ('{build_name}-{build_type}', '{job-name}'),
--- a/testing/mozharness/configs/routes.json
+++ b/testing/mozharness/configs/routes.json
@@ -10,10 +10,19 @@
         "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
         "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
     ],
     "l10n": [
         "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
         "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
         "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
+    ],
+    "nightly-l10n": [
+        "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
+        "{index}.gecko.v2.{project}.nightly.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
     ]
 }