Bug 1312000 - Sign Linux64, Linux32, and Android l10n desktop nightly repacks to TaskCluster (on date). r=kmoir
MozReview-Commit-ID: JGmZNO9oX7X
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.signing:SigningTask
+
+transforms:
+ - taskgraph.transforms.nightly_l10n_signing:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - nightly-l10n
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -14,23 +14,26 @@ class SigningTask(transform.TransformTas
We use a dictionary to create the input to the transforms.
It will have added to it keys `build-label`, the label for the build task,
and `build-platform` / `build-type`, its platform and type.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
- if config.get('kind-dependencies', []) != ["build"]:
+ if (config.get('kind-dependencies', []) != ["build"] and
+ config.get('kind-dependencies', []) != ["nightly-l10n"]):
raise Exception("Signing kinds must depend on builds")
for task in loaded_tasks:
- if task.kind != 'build':
+ if task.kind not in config.get('kind-dependencies'):
continue
if not task.attributes.get('nightly'):
continue
signing_task = {}
signing_task['build-label'] = task.label
signing_task['build-platform'] = task.attributes.get('build_platform')
signing_task['build-type'] = task.attributes.get('build_type')
signing_task['build-description'] = signing_task['build-platform']
signing_task['build-run-on-projects'] = task.attributes.get('run_on_projects')
signing_task['build-treeherder'] = {}
+ if task.attributes.get('l10n_chunk'):
+ signing_task['l10n_chunk'] = task.attributes['l10n_chunk']
yield signing_task
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -23,17 +23,16 @@ JOB_NAME_WHITELIST = set([
'android-api-15-nightly-opt',
'android-api-15-partner-sample1-opt',
'android-l10n-opt',
'android-nightly-l10n-opt',
'android-x86-opt',
'aries-debug',
'aries-eng-opt',
'browser-haz-debug',
- 'linux32-l10n-opt',
'linux64-artifact-opt',
'linux64-asan-debug',
'linux64-asan-opt',
'linux64-ccov-opt',
'linux64-debug',
'linux64-jsdcov-opt',
'linux64-l10n-opt',
'linux64-nightly-opt',
@@ -41,16 +40,17 @@ JOB_NAME_WHITELIST = set([
'linux64-opt',
'linux64-pgo',
'linux64-st-an-opt',
'linux64-valgrind-opt',
'linux-debug',
'linux-opt',
'linux-pgo',
'linux32-nightly-opt',
+ 'linux32-l10n-opt',
'linux32-nightly-l10n-opt',
'macosx64-debug',
'macosx64-opt',
'macosx64-st-an-opt',
'mulet-dbg',
'mulet-haz-debug',
'mulet-opt',
'nexus-5-l-eng-debug',
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -37,16 +37,17 @@ def chunkify(config, jobs):
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['run']['options'] = chunked['run'].get('options', [])
chunked['run']['options'].extend(["total-chunks={}".format(chunks),
"this-chunk={}".format(this_chunk)])
+ chunked['attributes']['l10n_chunk'] = this_chunk
yield chunked
else:
yield job
@transforms.add
def mh_config_replace_project(config, jobs):
""" Replaces {project} in mh config entries with the current project """
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -0,0 +1,147 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from mozbuild import milestone
+from taskgraph.transforms.base import TransformSequence
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/public/build/{}'
+
+transforms = TransformSequence()
+
+# XXXCallek: Do not merge this logic to m-c, its hardcoded for simplicity on date
+# And should actually parse chunking info and/or the locales file instead.
+LOCALES_PER_CHUNK = {
+ 'android': [
+ ['ar', 'be', 'ca', 'cs', 'da', 'de', 'es-AR', 'es-ES'], # chunk 1
+ ['et', 'eu', 'fa', 'fi', 'fr', 'fy-NL', 'ga-IE'], # chunk 2
+ ['gd', 'gl', 'he', 'hu', 'id', 'it', 'ja'], # chunk 3
+ ['ko', 'lt', 'nb-NO', 'nl', 'nn-NO', 'pa-IN', 'pl'], # chunk 4
+ ['pt-BR', 'pt-PT', 'ro', 'ru', 'sk', 'sl', 'sq'], # chunk 5
+ ['sr', 'sv-SE', 'th', 'tr', 'uk', 'zh-CN', 'zh-TW'], # chunk 6
+ ],
+ 'desktop': [
+ ['ar', 'ast', 'cs', 'de', 'en-GB', 'eo', 'es-AR'], # chunk 1
+ ['es-CL', 'es-ES', 'es-MX', 'fa', 'fr', 'fy-NL', 'gl'], # chunk 2
+ ['he', 'hu', 'id', 'it', 'ja', 'kk', 'ko'], # chunk 3
+ ['lt', 'lv', 'nb-NO', 'nl', 'nn-NO', 'pl'], # chunk 4
+ ['pt-BR', 'pt-PT', 'ru', 'sk', 'sl', 'sv-SE'], # chunk 5
+ ['th', 'tr', 'uk', 'vi', 'zh-CN', 'zh-TW'], # chunk 6
+ ]
+}
+
+# XXX Prettynames are bad, fix them
+PRETTYNAMES = {
+ 'desktop': "firefox-{version}.{locale}.{platform}.tar.bz2",
+ 'android': "fennec-{version}.{locale}.{platform}.apk",
+}
+PRETTY_PLATFORM_FROM_BUILD_PLATFORM = {
+ 'android-api-15-nightly': 'android-arm',
+ 'linux64-nightly': 'linux-x86_64',
+ 'linux-nightly': 'linux-i686',
+}
+_version_cache = None # don't get this multiple times
+
+
+def get_locale_list(product, chunk):
+ """ Gets the list of locales for this l10n chunk """
+ # XXXCallek This should be refactored to parse the locales file instead of
+ # a hardcoded list of things
+ if product not in LOCALES_PER_CHUNK.keys():
+ raise ValueError("Unexpected product passed")
+ return LOCALES_PER_CHUNK[product][chunk - 1]
+
+
+def get_version_number():
+ global _version_cache # Cache this value
+ if _version_cache:
+ return _version_cache
+ milestone_file = os.path.join('config', 'milestone.txt')
+ _version_cache = milestone.get_official_milestone(milestone_file)
+ return _version_cache
+
+
+def make_pretty_name(product, build_platform, locale):
+ # If this fails, we need to add a new key to PRETTY_PLATFORM_FROM_BUILD_PLATFORM
+ platform = PRETTY_PLATFORM_FROM_BUILD_PLATFORM[build_platform]
+ return PRETTYNAMES[product].format(
+ version=get_version_number(),
+ locale=locale,
+ platform=platform,
+ )
+
+
+@transforms.add
+def add_signing_artifacts(config, tasks):
+ for task in tasks:
+ task['unsigned-artifacts'] = []
+ product = 'android' if 'android' in task['build-platform'] else 'desktop'
+ for locale in get_locale_list(product, task['l10n_chunk']):
+ filename = make_pretty_name(product, task['build-platform'], locale)
+ task['unsigned-artifacts'].append({
+ 'task-reference': ARTIFACT_URL.format('unsigned-repack',
+ filename)
+ })
+ if 'tar.bz2' == filename[-7:]:
+ # Add the checksums file to be signed for linux
+ checksums_file = filename[:-7] + "checksums"
+ task['unsigned-artifacts'].append({
+ 'task-reference': ARTIFACT_URL.format('unsigned-repack',
+ checksums_file)
+ })
+ yield task
+
+
+@transforms.add
+def make_task_description(config, tasks):
+ for task in tasks:
+ task['label'] = task['build-label'].replace("nightly-l10n-", "signing-l10n-")
+ task['description'] = (task['build-description'].replace("-", " ") +
+ " l10n repack signing").title()
+ task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+
+ unsigned_artifacts = task['unsigned-artifacts']
+
+ task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
+ task['worker'] = {'implementation': 'scriptworker-signing',
+ 'unsigned-artifacts': unsigned_artifacts}
+
+ signing_format = "gpg" if "linux" in task['label'] else "jar"
+ signing_format_scope = "project:releng:signing:format:" + signing_format
+ task['scopes'] = ["project:releng:signing:cert:nightly-signing",
+ signing_format_scope]
+
+ task['dependencies'] = {'unsigned-repack': task['build-label']}
+ attributes = task.setdefault('attributes', {})
+ attributes['nightly'] = True
+ attributes['build_platform'] = task['build-platform']
+ attributes['build_type'] = task['build-type']
+ task['run-on-projects'] = task['build-run-on-projects']
+ task['treeherder'] = task['build-treeherder']
+ task['treeherder'].setdefault('symbol', 'tc(Ns{})'.format(
+ task.get('l10n_chunk', "")
+ ))
+ th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
+ th_platform = th_platform.replace("linux/opt", "linux32/opt")
+ task['treeherder'].setdefault('platform', th_platform)
+ task['treeherder'].setdefault('tier', 2)
+ task['treeherder'].setdefault('kind', 'build')
+
+ # delete stuff that's not part of a task description
+ del task['build-description']
+ del task['build-label']
+ del task['build-type']
+ del task['build-platform']
+ del task['build-run-on-projects']
+ del task['build-treeherder']
+ del task['l10n_chunk']
+ del task['unsigned-artifacts']
+
+ yield task