Bug 1312500 - Improve Signing Task reusability/comprehension. r=kmoir r=dustin
MozReview-Commit-ID: BVPcLN1uxn8
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,12 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: taskgraph.task.signing:SigningTask
transforms:
+ - taskgraph.transforms.build_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
--- a/taskcluster/ci/nightly-l10n-signing/kind.yml
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -1,12 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: taskgraph.task.signing:SigningTask
transforms:
- taskgraph.transforms.nightly_l10n_signing:transforms
+ - taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -185,14 +185,29 @@ implementation has its own section of th
expects. Thus the transforms that produce a task description must be aware of
the worker implementation to be used, but need not be aware of the details of
its payload format.
The ``task.py`` file also contains a dictionary mapping treeherder groups to
group names using an internal list of group names. Feel free to add additional
groups to this list as necessary.
+Signing Descriptions
+--------------------
+
+Signing kinds are passed a single dependent job (from its kind dependency) to act
+on.
+
+The transforms in ``taskcluster/taskgraph/transforms/signing.py`` implement
+this common functionality. They expect a "signing description", and produce a
+task definition. The schema for a signing description is defined at the top of
+``signing.py``, with copious comments.
+
+In particular you define a set of unsigned artifact urls (that point at the dependent
+task) and can optionally provide a dependent name (defaults to build) for use in
+task-reference. You also need to provide the signing format to use.
+
More Detail
-----------
The source files provide lots of additional detail, both in the code itself and
in the comments and docstrings. For the next level of detail beyond this file,
consult the transform source under ``taskcluster/taskgraph/transforms``.
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -22,18 +22,11 @@ class SigningTask(transform.TransformTas
if (config.get('kind-dependencies', []) != ["build"] and
config.get('kind-dependencies', []) != ["nightly-l10n"]):
raise Exception("Signing kinds must depend on builds")
for task in loaded_tasks:
if task.kind not in config.get('kind-dependencies'):
continue
if not task.attributes.get('nightly'):
continue
- signing_task = {}
- signing_task['build-label'] = task.label
- signing_task['build-platform'] = task.attributes.get('build_platform')
- signing_task['build-type'] = task.attributes.get('build_type')
- signing_task['build-description'] = signing_task['build-platform']
- signing_task['build-run-on-projects'] = task.attributes.get('run_on_projects')
- signing_task['build-treeherder'] = {}
- if task.attributes.get('l10n_chunk'):
- signing_task['l10n_chunk'] = task.attributes['l10n_chunk']
+ signing_task = {'dependent-task': task}
+
yield signing_task
copy from taskcluster/taskgraph/transforms/signing.py
copy to taskcluster/taskgraph/transforms/build_signing.py
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -10,57 +10,28 @@ from __future__ import absolute_import,
from taskgraph.transforms.base import TransformSequence
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
transforms = TransformSequence()
@transforms.add
-def make_task_description(config, tasks):
- for task in tasks:
- task['label'] = task['build-label'].replace("build-", "signing-")
- task['description'] = (task['build-description'].replace("-", " ") + " signing").title()
- task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+def make_signing_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+
+ job['label'] = dep_job.label.replace("build-", "signing-")
artifacts = []
- if 'android' in task['build-platform']:
+ if 'android' in dep_job.attributes.get('build_platform'):
artifacts = ['public/build/target.apk', 'public/build/en-US/target.apk']
else:
artifacts = ['public/build/target.tar.bz2']
unsigned_artifacts = []
for artifact in artifacts:
url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
unsigned_artifacts.append(url)
- task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
- task['worker'] = {'implementation': 'scriptworker-signing',
- 'unsigned-artifacts': unsigned_artifacts}
-
- signing_format = "gpg" if "linux" in task['label'] else "jar"
- signing_format_scope = "project:releng:signing:format:" + signing_format
- task['scopes'] = ["project:releng:signing:cert:nightly-signing",
- signing_format_scope]
+ job['unsigned-artifacts'] = unsigned_artifacts
+ job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
- task['dependencies'] = {'build': task['build-label']}
- attributes = task.setdefault('attributes', {})
- attributes['nightly'] = True
- attributes['build_platform'] = task['build-platform']
- attributes['build_type'] = task['build-type']
- task['run-on-projects'] = task['build-run-on-projects']
- task['treeherder'] = task['build-treeherder']
- task['treeherder'].setdefault('symbol', 'tc(Ns)')
- th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
- th_platform = th_platform.replace("linux/opt", "linux32/opt")
- th_platform = th_platform.replace("android-api-15/opt", "android-4-0-armv7-api15/opt")
- task['treeherder'].setdefault('platform', th_platform)
- task['treeherder'].setdefault('tier', 2)
- task['treeherder'].setdefault('kind', 'build')
-
- # delete stuff that's not part of a task description
- del task['build-description']
- del task['build-label']
- del task['build-type']
- del task['build-platform']
- del task['build-run-on-projects']
- del task['build-treeherder']
-
- yield task
+ yield job
--- a/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -74,74 +74,48 @@ def make_pretty_name(product, build_plat
return PRETTYNAMES[product].format(
version=get_version_number(),
locale=locale,
platform=platform,
)
@transforms.add
-def add_signing_artifacts(config, tasks):
- for task in tasks:
- task['unsigned-artifacts'] = []
- product = 'android' if 'android' in task['build-platform'] else 'desktop'
- for locale in get_locale_list(product, task['l10n_chunk']):
- filename = make_pretty_name(product, task['build-platform'], locale)
- task['unsigned-artifacts'].append({
+def add_signing_artifacts(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
+ dep_platform = dep_job.attributes.get('build_platform')
+
+ job['unsigned-artifacts'] = []
+ product = 'android' if 'android' in dep_platform else 'desktop'
+ l10n_chunk = dep_job.attributes.get('l10n_chunk')
+ for locale in get_locale_list(product, l10n_chunk):
+ filename = make_pretty_name(product, dep_platform, locale)
+ job['unsigned-artifacts'].append({
'task-reference': ARTIFACT_URL.format('unsigned-repack',
filename)
})
if 'tar.bz2' == filename[-7:]:
# Add the checksums file to be signed for linux
checksums_file = filename[:-7] + "checksums"
- task['unsigned-artifacts'].append({
+ job['unsigned-artifacts'].append({
'task-reference': ARTIFACT_URL.format('unsigned-repack',
checksums_file)
})
- yield task
+ yield job
@transforms.add
-def make_task_description(config, tasks):
- for task in tasks:
- task['label'] = task['build-label'].replace("nightly-l10n-", "signing-l10n-")
- task['description'] = (task['build-description'].replace("-", " ") +
- " l10n repack signing").title()
- task['description'] = task['description'].replace("Api 15", "4.0 API15+")
-
- unsigned_artifacts = task['unsigned-artifacts']
+def make_signing_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
- task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
- task['worker'] = {'implementation': 'scriptworker-signing',
- 'unsigned-artifacts': unsigned_artifacts}
-
- signing_format = "gpg" if "linux" in task['label'] else "jar"
- signing_format_scope = "project:releng:signing:format:" + signing_format
- task['scopes'] = ["project:releng:signing:cert:nightly-signing",
- signing_format_scope]
+ job['label'] = dep_job.label.replace("nightly-l10n-", "signing-l10n-")
- task['dependencies'] = {'unsigned-repack': task['build-label']}
- attributes = task.setdefault('attributes', {})
- attributes['nightly'] = True
- attributes['build_platform'] = task['build-platform']
- attributes['build_type'] = task['build-type']
- task['run-on-projects'] = task['build-run-on-projects']
- task['treeherder'] = task['build-treeherder']
- task['treeherder'].setdefault('symbol', 'tc(Ns{})'.format(
- task.get('l10n_chunk', "")
- ))
- th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
- th_platform = th_platform.replace("linux/opt", "linux32/opt")
- task['treeherder'].setdefault('platform', th_platform)
- task['treeherder'].setdefault('tier', 2)
- task['treeherder'].setdefault('kind', 'build')
+ job['depname'] = 'unsigned-repack'
+ job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
- # delete stuff that's not part of a task description
- del task['build-description']
- del task['build-label']
- del task['build-type']
- del task['build-platform']
- del task['build-run-on-projects']
- del task['build-treeherder']
- del task['l10n_chunk']
- del task['unsigned-artifacts']
-
- yield task
+ job['treeherder'] = {
+ # Format symbol appropriate for l10n chunking
+ 'symbol': 'tc-L10n(Ns{})'.format(
+ dep_job.attributes.get('l10n_chunk')),
+ }
+ yield job
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -2,65 +2,99 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
-from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.base import (
+ validate_schema,
+ TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
transforms = TransformSequence()
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+ basestring,
+ {Required('task-reference'): basestring})
+
+signing_description_schema = Schema({
+ # the dependant task (object) for this signing job, used to inform signing.
+ Required('dependent-task'): object,
+
+ # Artifacts from dep task to sign
+ Required('unsigned-artifacts'): [taskref_or_string],
+
+ # depname is used in taskref's to identify the taskID of the unsigned things
+ Required('depname', default='build'): basestring,
+
+ # Format to use to sign the artifacts
+ Required('signing-format'): basestring,
+
+ # unique label to describe this signing task, defaults to {dep.label}-signing
+ Optional('label'): basestring,
+
+ # treeherder is allowed here to override any defaults we use for signing. See
+ # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+ # below transforms for defaults of various values.
+ Optional('treeherder'): task_description_schema['treeherder'],
+})
+
@transforms.add
-def make_task_description(config, tasks):
- for task in tasks:
- task['label'] = task['build-label'].replace("build-", "signing-")
- task['description'] = (task['build-description'].replace("-", " ") + " signing").title()
- task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+def validate(config, jobs):
+ for job in jobs:
+ label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+ yield validate_schema(
+ signing_description_schema, job,
+ "In signing ({!r} kind) task for {!r}:".format(config.kind, label))
+
- artifacts = []
- if 'android' in task['build-platform']:
- artifacts = ['public/build/target.apk', 'public/build/en-US/target.apk']
- else:
- artifacts = ['public/build/target.tar.bz2']
- unsigned_artifacts = []
- for artifact in artifacts:
- url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
- unsigned_artifacts.append(url)
+@transforms.add
+def make_task_description(config, jobs):
+ for job in jobs:
+ dep_job = job['dependent-task']
- task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
- task['worker'] = {'implementation': 'scriptworker-signing',
- 'unsigned-artifacts': unsigned_artifacts}
+ signing_format_scope = "project:releng:signing:format:{}".format(
+ job['signing-format'])
- signing_format = "gpg" if "linux" in task['label'] else "jar"
- signing_format_scope = "project:releng:signing:format:" + signing_format
- task['scopes'] = ["project:releng:signing:cert:nightly-signing",
- signing_format_scope]
+ treeherder = job.get('treeherder', {})
+ treeherder.setdefault('symbol', 'tc(Ns)')
+ dep_th_platform = dep_job.task.get('extra', {}).get(
+ 'treeherder', {}).get('machine', {}).get('platform', '')
+ treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
+ treeherder.setdefault('tier', 2)
+ treeherder.setdefault('kind', 'build')
+
+ label = job.get('label', "{}-signing".format(dep_job))
- task['dependencies'] = {'build': task['build-label']}
- attributes = task.setdefault('attributes', {})
- attributes['nightly'] = True
- attributes['build_platform'] = task['build-platform']
- attributes['build_type'] = task['build-type']
- task['run-on-projects'] = task['build-run-on-projects']
- task['treeherder'] = task['build-treeherder']
- task['treeherder'].setdefault('symbol', 'tc(Ns)')
- th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
- th_platform = th_platform.replace("linux/opt", "linux32/opt")
- th_platform = th_platform.replace("android-api-15/opt", "android-4-0-armv7-api15/opt")
- task['treeherder'].setdefault('platform', th_platform)
- task['treeherder'].setdefault('tier', 2)
- task['treeherder'].setdefault('kind', 'build')
-
- # delete stuff that's not part of a task description
- del task['build-description']
- del task['build-label']
- del task['build-type']
- del task['build-platform']
- del task['build-run-on-projects']
- del task['build-treeherder']
+ task = {
+ 'label': label,
+ 'description': "{} Signing".format(
+ dep_job.task["metadata"]["description"]),
+ 'worker-type': "scriptworker-prov-v1/signing-linux-v1",
+ 'worker': {'implementation': 'scriptworker-signing',
+ 'unsigned-artifacts': job['unsigned-artifacts']},
+ 'scopes': ["project:releng:signing:cert:nightly-signing",
+ signing_format_scope],
+ 'dependencies': {job['depname']: dep_job.label},
+ 'attributes': {
+ 'nightly': dep_job.attributes.get('nightly', False),
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ },
+ 'run-on-projects': dep_job.attributes.get('run_on_projects'),
+ 'treeherder': treeherder,
+ }
yield task