Bug 1312000 - Sign Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster (on date). r=kmoir
authorJustin Wood <Callek@gmail.com>
Wed, 19 Oct 2016 09:03:21 -0400
changeset 428079 79a2f66ff5c00661f943ff2c5107370008ba7ed8
parent 428078 0eb1a4a50d9a5601a652c0e832b66b7b0e56228e
child 428931 b5d40cc4aefc5180282157dc8146d48b8d657120
child 429397 7bf51d704206b3b785f43eceb6c6887f5a6445ce
push id33223
push userCallek@gmail.com
push dateFri, 21 Oct 2016 14:42:19 +0000
reviewerskmoir
bugs1312000
milestone52.0a1
Bug 1312000 - Sign Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster (on date). r=kmoir MozReview-Commit-ID: JGmZNO9oX7X
taskcluster/ci/nightly-l10n-signing/kind.yml
taskcluster/taskgraph/task/signing.py
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/nightly_l10n_signing.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.signing:SigningTask
+
+transforms:
+   - taskgraph.transforms.nightly_l10n_signing:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -14,23 +14,26 @@ class SigningTask(transform.TransformTas
 
     We use a dictionary to create the input to the transforms.
     It will have added to it keys `build-label`, the label for the build task,
     and `build-platform` / `build-type`, its platform and type.
     """
 
     @classmethod
     def get_inputs(cls, kind, path, config, params, loaded_tasks):
-        if config.get('kind-dependencies', []) != ["build"]:
+        if (config.get('kind-dependencies', []) != ["build"] and
+                config.get('kind-dependencies', []) != ["nightly-l10n"]):
             raise Exception("Signing kinds must depend on builds")
         for task in loaded_tasks:
-            if task.kind != 'build':
+            if task.kind not in config.get('kind-dependencies'):
                 continue
             if not task.attributes.get('nightly'):
                 continue
             signing_task = {}
             signing_task['build-label'] = task.label
             signing_task['build-platform'] = task.attributes.get('build_platform')
             signing_task['build-type'] = task.attributes.get('build_type')
             signing_task['build-description'] = signing_task['build-platform']
             signing_task['build-run-on-projects'] = task.attributes.get('run_on_projects')
             signing_task['build-treeherder'] = {}
+            if task.attributes.get('l10n_chunk'):
+                signing_task['l10n_chunk'] = task.attributes['l10n_chunk']
             yield signing_task
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -23,17 +23,16 @@ JOB_NAME_WHITELIST = set([
     'android-api-15-nightly-opt',
     'android-api-15-partner-sample1-opt',
     'android-l10n-opt',
     'android-nightly-l10n-opt',
     'android-x86-opt',
     'aries-debug',
     'aries-eng-opt',
     'browser-haz-debug',
-    'linux32-l10n-opt',
     'linux64-artifact-opt',
     'linux64-asan-debug',
     'linux64-asan-opt',
     'linux64-ccov-opt',
     'linux64-debug',
     'linux64-jsdcov-opt',
     'linux64-l10n-opt',
     'linux64-nightly-opt',
@@ -41,16 +40,17 @@ JOB_NAME_WHITELIST = set([
     'linux64-opt',
     'linux64-pgo',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
     'linux-debug',
     'linux-opt',
     'linux-pgo',
     'linux32-nightly-opt',
+    'linux32-l10n-opt',
     'linux32-nightly-l10n-opt',
     'macosx64-debug',
     'macosx64-opt',
     'macosx64-st-an-opt',
     'mulet-dbg',
     'mulet-haz-debug',
     'mulet-opt',
     'nexus-5-l-eng-debug',
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -37,16 +37,17 @@ def chunkify(config, jobs):
             for this_chunk in range(1, chunks + 1):
                 chunked = copy.deepcopy(job)
                 chunked['name'] = chunked['name'].replace(
                     '/', '-{}/'.format(this_chunk), 1
                 )
                 chunked['run']['options'] = chunked['run'].get('options', [])
                 chunked['run']['options'].extend(["total-chunks={}".format(chunks),
                                                   "this-chunk={}".format(this_chunk)])
+                chunked['attributes']['l10n_chunk'] = this_chunk
                 yield chunked
         else:
             yield job
 
 
 @transforms.add
 def mh_config_replace_project(config, jobs):
     """ Replaces {project} in mh config entries with the current project """
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -0,0 +1,147 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from mozbuild import milestone
+from taskgraph.transforms.base import TransformSequence
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/public/build/{}'
+
+transforms = TransformSequence()
+
+# XXXCallek: Do not merge this logic to m-c, its hardcoded for simplicity on date
+# And should actually parse chunking info and/or the locales file instead.
+LOCALES_PER_CHUNK = {
+    'android': [
+        ['ar', 'be', 'ca', 'cs', 'da', 'de', 'es-AR', 'es-ES'],  # chunk 1
+        ['et', 'eu', 'fa', 'fi', 'fr', 'fy-NL', 'ga-IE'],  # chunk 2
+        ['gd', 'gl', 'he', 'hu', 'id', 'it', 'ja'],  # chunk 3
+        ['ko', 'lt', 'nb-NO', 'nl', 'nn-NO', 'pa-IN', 'pl'],  # chunk 4
+        ['pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq'],  # chunk 5
+        ['sr', 'sv-SE', 'th', 'tr', 'uk', 'zh-CN', 'zh-TW'],  # chunk 6
+    ],
+    'desktop': [
+        ['ar', 'ast', 'cs', 'de', 'en-GB', 'eo', 'es-AR'],  # chunk 1
+        ['es-CL', 'es-ES', 'es-MX', 'fa', 'fr', 'fy-NL', 'gl'],  # chunk 2
+        ['he', 'hu', 'id', 'it', 'ja', 'kk', 'ko'],  # chunk 3
+        ['lt', 'lv', 'nb-NO', 'nl', 'nn-NO', 'pl'],  # chunk 4
+        ['pt-BR', 'pt-PT', 'ru', 'sk', 'sl', 'sv-SE'],  # chunk 5
+        ['th', 'tr', 'uk', 'vi', 'zh-CN', 'zh-TW'],  # chunk 6
+    ]
+}
+
+# XXX Prettynames are bad, fix them
+PRETTYNAMES = {
+    'desktop': "firefox-{version}.{locale}.{platform}.tar.bz2",
+    'android': "fennec-{version}.{locale}.{platform}.apk",
+}
+PRETTY_PLATFORM_FROM_BUILD_PLATFORM = {
+    'android-api-15-nightly': 'android-arm',
+    'linux64-nightly': 'linux-x86_64',
+    'linux-nightly': 'linux-i686',
+}
+_version_cache = None  # don't get this multiple times
+
+
+def get_locale_list(product, chunk):
+    """ Gets the list of locales for this l10n chunk """
+    # XXXCallek This should be refactored to parse the locales file instead of
+    # a hardcoded list of things
+    if product not in LOCALES_PER_CHUNK.keys():
+        raise ValueError("Unexpected product passed")
+    return LOCALES_PER_CHUNK[product][chunk - 1]
+
+
+def get_version_number():
+    global _version_cache  # Cache this value
+    if _version_cache:
+        return _version_cache
+    milestone_file = os.path.join('config', 'milestone.txt')
+    _version_cache = milestone.get_official_milestone(milestone_file)
+    return _version_cache
+
+
+def make_pretty_name(product, build_platform, locale):
+    # If this fails, we need to add a new key to PRETTY_PLATFORM_FROM_BUILD_PLATFORM
+    platform = PRETTY_PLATFORM_FROM_BUILD_PLATFORM[build_platform]
+    return PRETTYNAMES[product].format(
+        version=get_version_number(),
+        locale=locale,
+        platform=platform,
+    )
+
+
+@transforms.add
+def add_signing_artifacts(config, tasks):
+    for task in tasks:
+        task['unsigned-artifacts'] = []
+        product = 'android' if 'android' in task['build-platform'] else 'desktop'
+        for locale in get_locale_list(product, task['l10n_chunk']):
+            filename = make_pretty_name(product, task['build-platform'], locale)
+            task['unsigned-artifacts'].append({
+                'task-reference': ARTIFACT_URL.format('unsigned-repack',
+                                                      filename)
+                })
+            if 'tar.bz2' == filename[-7:]:
+                # Add the checksums file to be signed for linux
+                checksums_file = filename[:-7] + "checksums"
+                task['unsigned-artifacts'].append({
+                    'task-reference': ARTIFACT_URL.format('unsigned-repack',
+                                                          checksums_file)
+                    })
+        yield task
+
+
+@transforms.add
+def make_task_description(config, tasks):
+    for task in tasks:
+        task['label'] = task['build-label'].replace("nightly-l10n-", "signing-l10n-")
+        task['description'] = (task['build-description'].replace("-", " ") +
+                               " l10n repack signing").title()
+        task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+
+        unsigned_artifacts = task['unsigned-artifacts']
+
+        task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
+        task['worker'] = {'implementation': 'scriptworker-signing',
+                          'unsigned-artifacts': unsigned_artifacts}
+
+        signing_format = "gpg" if "linux" in task['label'] else "jar"
+        signing_format_scope = "project:releng:signing:format:" + signing_format
+        task['scopes'] = ["project:releng:signing:cert:nightly-signing",
+                          signing_format_scope]
+
+        task['dependencies'] = {'unsigned-repack': task['build-label']}
+        attributes = task.setdefault('attributes', {})
+        attributes['nightly'] = True
+        attributes['build_platform'] = task['build-platform']
+        attributes['build_type'] = task['build-type']
+        task['run-on-projects'] = task['build-run-on-projects']
+        task['treeherder'] = task['build-treeherder']
+        task['treeherder'].setdefault('symbol', 'tc(Ns{})'.format(
+            task.get('l10n_chunk', "")
+        ))
+        th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
+        th_platform = th_platform.replace("linux/opt", "linux32/opt")
+        task['treeherder'].setdefault('platform', th_platform)
+        task['treeherder'].setdefault('tier', 2)
+        task['treeherder'].setdefault('kind', 'build')
+
+        # delete stuff that's not part of a task description
+        del task['build-description']
+        del task['build-label']
+        del task['build-type']
+        del task['build-platform']
+        del task['build-run-on-projects']
+        del task['build-treeherder']
+        del task['l10n_chunk']
+        del task['unsigned-artifacts']
+
+        yield task