Bug 1351387: repackage for OSX l10n. r=kmoir
authorJustin Wood <Callek@gmail.com>
Fri, 07 Apr 2017 09:26:35 -0400
changeset 560572 59ee008c9b230aa94087ce8ab9bcd1ca68a61c80
parent 560571 f9570cd8b8214cf07485208cbdfbb2d95760600d
child 561486 996051191161e993559f824d855b90c609271715
push id53457
push userCallek@gmail.com
push dateTue, 11 Apr 2017 16:23:24 +0000
reviewerskmoir
bugs1351387
milestone55.0a1
Bug 1351387: repackage for OSX l10n. r=kmoir MozReview-Commit-ID: AIH3HRYokQL
taskcluster/ci/repackage-l10n/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/repackage.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_l10n.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.repackage:loader
+
+transforms:
+   - taskgraph.transforms.repackage_l10n:transforms
+   - taskgraph.transforms.repackage:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n-signing
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -208,8 +208,16 @@ in order to make the decision.
 push-apk
 --------
 PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
 all at once. They also depend on the breakpoint.
 
 repackage
 ---------
+Repackage tasks take a signed output and package them up into something suiteable
+for shipping to our users. For example, on OSX we return a tarball as the signed output
+and this task would package that up as an Apple Disk Image (.dmg)
+
+
+repackage-l10n
+--------------
+Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks.
--- a/taskcluster/taskgraph/loader/repackage.py
+++ b/taskcluster/taskgraph/loader/repackage.py
@@ -11,17 +11,18 @@ logger = logging.getLogger(__name__)
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Generate tasks implementing repackage jobs.  These depend on build jobs
     and generate the package in the correct format after being signed by the
     build job
     """
 
-    if (config.get('kind-dependencies', []) != ['build-signing']):
+    if (config.get('kind-dependencies', []) != ['build-signing']) and \
+       (config.get('kind-dependencies', []) != ['nightly-l10n-signing']):
         raise Exception("Repackage signing tasks must depend on build-signing tasks")
     for task in loaded_tasks:
         if not task.attributes.get('nightly'):
             continue
         if task.kind not in config.get('kind-dependencies'):
             continue
         build_platform = task.attributes.get('build_platform')
         build_type = task.attributes.get('build_type')
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -35,16 +35,19 @@ packaging_description_schema = Schema({
     # unique label to describe this signing task, defaults to {dep.label}-signing
     Optional('label'): basestring,
 
     # treeherder is allowed here to override any defaults we use for signing.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    # If a l10n task, the corresponding locale
+    Optional('locale'): basestring,
+
     # Routes specific to this task, if defined
     Optional('routes'): [basestring],
 })
 
 
 # comment out adding routes until talking to mshal
 # @transforms.add
 # def add_repackage_routes(config, jobs):
@@ -60,50 +63,49 @@ packaging_description_schema = Schema({
 #                continue
 #            branch = dep_route.split(".")[3]
 #            rest = ".".join(dep_route.split(".")[5:])
 #            job['routes'].append(
 #                'index.gecko.v2.{}.repackage-nightly.{}'.format(branch, rest))
 #        yield job
 
 @transforms.add
-def make_repackage_description(config, jobs):
-    for job in jobs:
-        dep_job = job['dependent-task']
-
-        label = dep_job.label.replace("signing-", "repackage-")
-        job['label'] = label
-
-        yield job
-
-
-@transforms.add
 def validate(config, jobs):
     for job in jobs:
         label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
         yield validate_schema(
             packaging_description_schema, job,
             "In packaging ({!r} kind) task for {!r}:".format(config.kind, label))
 
 
 @transforms.add
+def make_repackage_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        label = job.get('label',
+                        dep_job.label.replace("signing-", "repackage-"))
+        job['label'] = label
+
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'tc(Nr)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
-        label = job.get('label', "{}-repackage".format(dep_job.label))
-
         attributes = {
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
 
         command = ['/home/worker/bin/run-task',
                    # Various caches/volumes are default owned by root:root.
@@ -111,23 +113,28 @@ def make_task_description(config, jobs):
                    '--chown-recursive', '/home/worker/tooltool-cache',
                    '--vcs-checkout', '/home/worker/workspace/build/src',
                    '--tools-checkout', '/home/worker/workspace/build/tools',
                    '--',
                    '/home/worker/workspace/build/src/taskcluster/scripts/builder/repackage.sh'
                    ]
 
         dependencies = {dep_job.attributes.get('kind'): dep_job.label}
-        input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
-            'public/build/target.tar.gz'
+        if job.get('locale'):
+            input_string = 'https://queue.taskcluster.net/v1/task/<nightly-l10n-signing>/' + \
+                'artifacts/public/build/{}/target.tar.gz'
+            input_string = input_string.format(job['locale'])
+        else:
+            input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
+                'public/build/target.tar.gz'
         signed_input = {'task-reference': input_string}
         level = config.params['level']
 
         task = {
-            'label': label,
+            'label': job['label'],
             'description': "{} Repackage".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': 'aws-provisioner-v1/gecko-%s-b-macosx64' % level,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
             'routes': job.get('routes', []),
@@ -168,11 +175,11 @@ def make_task_description(config, jobs):
                                'MOZHARNESS_CONFIG': 'repackage/osx_signed.py',
                                'USE_SCCACHE': '1',
                                'MOZHARNESS_SCRIPT': 'mozharness/scripts/repackage.py'
                                },
                        'command': command,
                        'chain-of-trust': True,
                        'relengapi-proxy': True,
                        'max-run-time': 3600
-               }
+                       }
         }
         yield task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/repackage_l10n.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the repackage task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def split_locales(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        for locale in dep_job.attributes.get('chunk_locales', []):
+            job['locale'] = locale
+
+            label = dep_job.label.replace("signing-", "repackage-{}-".format(locale))
+            label = "repackage-{}-{}/{}".format(locale,
+                                                dep_job.attributes['build_platform'],
+                                                dep_job.attributes['build_type'],)
+            job['label'] = label
+
+            treeherder = job.get('treeherder', {})
+            treeherder['symbol'] = 'tc-L10n-Rpk({})'.format(locale)
+            job['treeherder'] = treeherder
+            yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -387,16 +387,17 @@ GROUP_NAMES = {
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'tc-cs': 'Checksum signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',