Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki
MozReview-Commit-ID: 1DEIjFYAx74
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -124,16 +124,21 @@ Signals whether the task is part of a ni
out nightly tasks from full task set at target stage.
all_locales
===========
For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list
of relevant locales for the platform.
+all_locales_with_changesets
+===========================
+
+Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).
+
l10n_chunk
==========
For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
number of the job. Note that this is a string!
chunk_locales
=============
For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Do transforms specific to l10n kind
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
+import json
from mozbuild.chunkify import chunkify
from taskgraph.transforms.base import (
TransformSequence,
)
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
@@ -131,24 +132,34 @@ l10n_description_schema = Schema({
transforms = TransformSequence()
def _parse_locales_file(locales_file, platform=None):
""" Parse the passed locales file for a list of locales.
If platform is unset matches all platforms.
"""
locales = []
- if locales_file.endswith('json'):
- # Release process uses .json for locale files sometimes.
- raise NotImplementedError("Don't know how to parse a .json locales file")
- else:
- with open(locales_file, mode='r') as lf:
- locales = lf.read().split()
+
+ with open(locales_file, mode='r') as f:
+ if locales_file.endswith('json'):
+ all_locales = json.load(f)
+ # XXX Only single locales are fetched
+ locales = {locale: data['revision'] for locale, data in all_locales.items() if 'android' in data['platforms']}
+ else:
+ all_locales = f.read().split()
+ # 'default' is the hg revision at the top of hg repo, in this context
+ locales = {locale: 'default' for locale in all_locales}
return locales
+def _remove_ja_jp_mac_locale(locales):
+ # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, so just omit it unconditionally
+ return {
+ locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'
+ }
+
@transforms.add
def setup_name(config, jobs):
for job in jobs:
dep = job['dependent-task']
if dep.attributes.get('nightly'):
# Set the name to the same as the dep task, without kind name.
# Label will get set automatically with this kinds name.
@@ -227,62 +238,62 @@ def handle_keyed_by(config, jobs):
for field in fields:
resolve_keyed_by(item=job, field=field, item_name=job['name'])
yield job
@transforms.add
def all_locales_attribute(config, jobs):
for job in jobs:
- locales = set(_parse_locales_file(job["locales-file"]))
- # ja-JP-mac is a mac-only locale, but there are no
- # mac builds being repacked, so just omit it unconditionally
- locales = locales - set(("ja-JP-mac", ))
- # Convert to mutable list.
- locales = list(sorted(locales))
+ locales_with_changesets = _parse_locales_file(job["locales-file"])
+ locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)
+
+ locales = sorted(locales_with_changesets.keys())
attributes = job.setdefault('attributes', {})
attributes["all_locales"] = locales
+ attributes["all_locales_with_changesets"] = locales_with_changesets
yield job
@transforms.add
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
- all_locales = job['attributes']['all_locales']
+ locales_with_changesets = job['attributes']['all_locales_with_changesets']
if chunks:
- if chunks > len(all_locales):
+ if chunks > len(locales_with_changesets):
# Reduce chunks down to the number of locales
- chunks = len(all_locales)
+ chunks = len(locales_with_changesets)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
- my_locales = []
- my_locales = chunkify(all_locales, this_chunk, chunks)
+ # chunkify doesn't work with dicts
+ locales_with_changesets_as_list = locales_with_changesets.items()
+ chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
chunked['mozharness']['options'].extend([
- "locale={}".format(locale) for locale in my_locales
- ])
+ 'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales
+ ])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
- chunked['attributes']['chunk_locales'] = my_locales
+ chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales] # strip revision
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
- "locale={}".format(locale) for locale in all_locales
- ])
+ 'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items()
+ ])
yield job
@transforms.add
def mh_config_replace_project(config, jobs):
""" Replaces {project} in mh config entries with the current project """
# XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
for job in jobs: