Bug 1319189 - Create tasks that move desktop l10n artifacts to the s3 final locations. r=kmoir draft
authorJustin Wood <Callek@gmail.com>
Mon, 21 Nov 2016 16:08:34 -0500
changeset 442154 7cd0fb550943b9d17d95e096af12e14d6c20fdbf
parent 442153 3640c828f461a8df880c13d9b61bd7a74ad2d48b
child 442162 6d971614e10d685a83069f964bd03a21fe85d474
push id36600
push userCallek@gmail.com
push dateMon, 21 Nov 2016 21:11:58 +0000
reviewerskmoir
bugs1319189
milestone53.0a1
Bug 1319189 - Create tasks that move desktop l10n artifacts to the s3 final locations. r=kmoir MozReview-Commit-ID: FXYtJwmjBmd
taskcluster/ci/beetmover-l10n/kind.yml
taskcluster/taskgraph/task/beetmover.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_l10n.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+   - taskgraph.transforms.beetmover_l10n:transforms
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n
+  - nightly-l10n-signing
--- a/taskcluster/taskgraph/task/beetmover.py
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -14,19 +14,20 @@ class BeetmoverTask(transform.TransformT
 
     We use a dictionary to create the input to the transforms.
     It will have added to it keys `build-label`, the label for the build task,
     and `build-platform` / `build-type`, its platform and type.
     """
 
     @classmethod
     def get_inputs(cls, kind, path, config, params, loaded_tasks):
-        if config.get('kind-dependencies', []) != ["build", "build-signing"]:
+        if config.get('kind-dependencies', []) != ["build", "build-signing"] and \
+           config.get('kind-dependencies', []) != ["nightly-l10n", "nightly-l10n-signing"]:
             raise Exception("Beetmover kinds must depend on builds or signing builds")
         for task in loaded_tasks:
             if not task.attributes.get('nightly'):
                 continue
-            if task.kind != "build-signing" and task.kind != "build":
+            if task.kind not in config.get('kind-dependencies'):
                 continue
             beetmover_task = {}
             beetmover_task['dependent-task'] = task
 
             yield beetmover_task
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -36,16 +36,19 @@ beetmover_description_schema = Schema({
 
     # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
     Optional('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for beetmover.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
 })
 
 
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
         yield validate_schema(
@@ -75,33 +78,41 @@ def make_task_description(config, jobs):
         dependent_kind = str(job['dependent-task'].kind)
         # both artifacts are the build artifacts if not the signing task
         taskid_of_manifest = "<" + str(dependent_kind) + ">"
         taskid_to_beetmove = taskid_of_manifest
         update_manifest = False
         dependencies = {job['dependent-task'].kind: dep_job.label}
         # taskid_of_manifest always refers to the unsigned task
         if "signing" in dependent_kind:
-            taskid_of_manifest = "<build>"
+            if len(dep_job.dependencies) > 1:
+                raise NotImplementedError(
+                    "can't beetmove a signing task with multiple dependencies")
+            dep_name = dep_job.dependencies.keys()[0]
+            taskid_of_manifest = "<" + str(dep_name) + ">"
             update_manifest = True
             signing_dependencies = dep_job.dependencies
             dependencies.update(signing_dependencies)
 
+        worker = {'implementation': 'beetmover',
+                  'taskid_to_beetmove': {"task-reference":
+                                         taskid_to_beetmove},
+                  'taskid_of_manifest': {"task-reference":
+                                         taskid_of_manifest},
+                  'update_manifest': update_manifest}
+        if job.get('locale'):
+            worker['locale'] = job['locale']
+
         task = {
             'label': label,
             'description': "{} Beetmover".format(
                 dep_job.task["metadata"]["description"]),
             # do we have to define worker type somewhere?
             'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
-            'worker': {'implementation': 'beetmover',
-                       'taskid_to_beetmove': {"task-reference":
-                                              taskid_to_beetmove},
-                       'taskid_of_manifest': {"task-reference":
-                                              taskid_of_manifest},
-                       'update_manifest': update_manifest},
+            'worker': worker,
             'scopes': [],
             'dependencies': dependencies,
             'attributes': {
                 'nightly': dep_job.attributes.get('nightly', False),
                 'build_platform': dep_job.attributes.get('build_platform'),
                 'build_type': dep_job.attributes.get('build_type'),
             },
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.treeherder import join_symbol
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def ignore_android_moving(config, jobs):
+    for job in jobs:
+        if 'android' in job['dependent-task'].label:
+            # skip android until we general balrog_props.json for it
+            continue
+        yield job
+
+
+@transforms.add
+def make_signing_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        for locale in dep_job.attributes.get('chunk_locales', []):
+
+            label = job.get('label',
+                            "beetmover-{}-{}".format(locale, dep_job.label))
+
+            # add the locale code
+            symbol = locale
+            group = 'tc-BM-L10n'
+
+            treeherder = {
+                'symbol': join_symbol(group, symbol),
+            }
+
+            beet_description = {
+                'dependent-task': dep_job,
+                'treeherder': treeherder,
+                'label': label,
+                'locale': locale,
+            }
+            yield beet_description
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -88,29 +88,34 @@ def make_task_description(config, jobs):
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 2)
         treeherder.setdefault('kind', 'build')
 
         label = job.get('label', "{}-signing".format(dep_job))
 
+        attributes = {
+                'nightly': dep_job.attributes.get('nightly', False),
+                'build_platform': dep_job.attributes.get('build_platform'),
+                'build_type': dep_job.attributes.get('build_type'),
+        }
+        if dep_job.attributes.get('chunk_locales'):
+            # Used for l10n attribute passthrough
+            attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
+
         task = {
             'label': label,
             'description': "{} Signing".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
             'scopes': ["project:releng:signing:cert:nightly-signing"] + \
                     signing_format_scopes,
             'dependencies': {job['depname']: dep_job.label},
-            'attributes': {
-                'nightly': dep_job.attributes.get('nightly', False),
-                'build_platform': dep_job.attributes.get('build_platform'),
-                'build_type': dep_job.attributes.get('build_type'),
-            },
+            'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
         yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -304,16 +304,19 @@ task_description_schema = Schema({
         # taskid of task with artifacts to beetmove
         Required('taskid_to_beetmove'): taskref_or_string,
 
         # taskid of task with artifacts to beetmove
         Required('taskid_of_manifest'): taskref_or_string,
 
         # beetmover template key
         Required('update_manifest'): bool,
+        
+        # locale key, if this is a locale beetmover job
+        Optional('locale'): basestring,
     }, {
         Required('implementation'): 'balrog',
 
         # taskid of the signed beetmoved task
         Required('task_artifact_url'): taskref_or_string,
     }),
 
     # The "when" section contains descriptions of the circumstances
@@ -339,16 +342,17 @@ GROUP_NAMES = {
     'tc-R': 'Reftests executed by TaskCluster',
     'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
@@ -510,18 +514,20 @@ def build_scriptworker_signing_payload(c
 def build_beetmover_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upload_date': config.params['build_date'],
         'taskid_to_beetmove': worker['taskid_to_beetmove'],
         'taskid_of_manifest': worker['taskid_of_manifest'],
-        'update_manifest': worker['update_manifest']
+        'update_manifest': worker['update_manifest'],
     }
+    if worker.get('locale'):
+        task_def['payload']['locale'] = worker['locale']
 
 
 @payload_builder('balrog')
 def build_balrog_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'parent_task_artifacts_url': worker['task_artifact_url'],