Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 14 Feb 2017 15:04:54 +0100
changeset 485929 cf5decf5c739b0b08a040338db7d5f052e953436
parent 485423 340c5eaf341c7dfdf2957b17da4a67999df92537
child 486048 03a23bffd9684a4fa9a7bd9d5d6c9dccf65547d4
push id45882
push userbmo:jlorenzo@mozilla.com
push dateFri, 17 Feb 2017 10:38:49 +0000
reviewersaki
bugs1337825
milestone54.0a1
Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki MozReview-Commit-ID: 1DEIjFYAx74
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/l10n.py
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -124,16 +124,21 @@ Signals whether the task is part of a ni
 out nightly tasks from full task set at target stage.
 
 all_locales
 ===========
 
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list
 of relevant locales for the platform.
 
+all_locales_with_changesets
+===========================
+
+Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).
+
 l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Do transforms specific to l10n kind
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import json
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import (
     TransformSequence,
 )
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
@@ -131,24 +132,34 @@ l10n_description_schema = Schema({
 transforms = TransformSequence()
 
 
 def _parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
         If platform is unset matches all platforms.
     """
     locales = []
-    if locales_file.endswith('json'):
-        # Release process uses .json for locale files sometimes.
-        raise NotImplementedError("Don't know how to parse a .json locales file")
-    else:
-        with open(locales_file, mode='r') as lf:
-            locales = lf.read().split()
+
+    with open(locales_file, mode='r') as f:
+        if locales_file.endswith('json'):
+            all_locales = json.load(f)
+            # XXX Only single locales are fetched
+            locales = {locale: data['revision'] for locale, data in all_locales.items() if 'android' in data['platforms']}
+        else:
+            all_locales = f.read().split()
+            # 'default' is the hg revision at the top of hg repo, in this context
+            locales = {locale: 'default' for locale in all_locales}
     return locales
 
+def _remove_ja_jp_mac_locale(locales):
+    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, so just omit it unconditionally
+    return {
+        locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'
+    }
+
 
 @transforms.add
 def setup_name(config, jobs):
     for job in jobs:
         dep = job['dependent-task']
         if dep.attributes.get('nightly'):
             # Set the name to the same as the dep task, without kind name.
             # Label will get set automatically with this kinds name.
@@ -227,62 +238,62 @@ def handle_keyed_by(config, jobs):
         for field in fields:
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
-        locales = set(_parse_locales_file(job["locales-file"]))
-        # ja-JP-mac is a mac-only locale, but there are no
-        # mac builds being repacked, so just omit it unconditionally
-        locales = locales - set(("ja-JP-mac", ))
-        # Convert to mutable list.
-        locales = list(sorted(locales))
+        locales_with_changesets = _parse_locales_file(job["locales-file"])
+        locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)
+
+        locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
+        attributes["all_locales_with_changesets"] = locales_with_changesets
         yield job
 
 
 @transforms.add
 def chunk_locales(config, jobs):
     """ Utilizes chunking for l10n stuff """
     for job in jobs:
         chunks = job.get('chunks')
-        all_locales = job['attributes']['all_locales']
+        locales_with_changesets = job['attributes']['all_locales_with_changesets']
         if chunks:
-            if chunks > len(all_locales):
+            if chunks > len(locales_with_changesets):
                 # Reduce chunks down to the number of locales
-                chunks = len(all_locales)
+                chunks = len(locales_with_changesets)
             for this_chunk in range(1, chunks + 1):
                 chunked = copy.deepcopy(job)
                 chunked['name'] = chunked['name'].replace(
                     '/', '-{}/'.format(this_chunk), 1
                 )
                 chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
-                my_locales = []
-                my_locales = chunkify(all_locales, this_chunk, chunks)
+                # chunkify doesn't work with dicts
+                locales_with_changesets_as_list = locales_with_changesets.items()
+                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
                 chunked['mozharness']['options'].extend([
-                    "locale={}".format(locale) for locale in my_locales
-                    ])
+                    'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales
+                ])
                 chunked['attributes']['l10n_chunk'] = str(this_chunk)
-                chunked['attributes']['chunk_locales'] = my_locales
+                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]  # strip revision
 
                 # add the chunk number to the TH symbol
                 group, symbol = split_symbol(
                     chunked.get('treeherder', {}).get('symbol', ''))
                 symbol += str(this_chunk)
                 chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                 yield chunked
         else:
             job['mozharness']['options'] = job['mozharness'].get('options', [])
             job['mozharness']['options'].extend([
-                "locale={}".format(locale) for locale in all_locales
-                ])
+                'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items()
+            ])
             yield job
 
 
 @transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
     # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
     for job in jobs: