Bug 1312500 - Improve Signing Task reusability/comprehension. r=kmoir r=dustin draft
authorJustin Wood <Callek@gmail.com>
Tue, 25 Oct 2016 15:58:57 -0400
changeset 429398 b782287509611e79d5c4a5d921145132e73308a1
parent 429397 7bf51d704206b3b785f43eceb6c6887f5a6445ce
child 429406 cf72deaddbada60c3f5dd673b1b05c98c6fed622
push id33552
push userCallek@gmail.com
push dateTue, 25 Oct 2016 20:04:22 +0000
reviewerskmoir, dustin
bugs1312500
milestone52.0a1
Bug 1312500 - Improve Signing Task reusability/comprehension. r=kmoir r=dustin MozReview-Commit-ID: BVPcLN1uxn8
taskcluster/ci/build-signing/kind.yml
taskcluster/ci/nightly-l10n-signing/kind.yml
taskcluster/docs/transforms.rst
taskcluster/taskgraph/task/signing.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/nightly_l10n_signing.py
taskcluster/taskgraph/transforms/signing.py
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 implementation: taskgraph.task.signing:SigningTask
 
 transforms:
+   - taskgraph.transforms.build_signing:transforms
    - taskgraph.transforms.signing:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - build
--- a/taskcluster/ci/nightly-l10n-signing/kind.yml
+++ b/taskcluster/ci/nightly-l10n-signing/kind.yml
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 implementation: taskgraph.task.signing:SigningTask
 
 transforms:
    - taskgraph.transforms.nightly_l10n_signing:transforms
+   - taskgraph.transforms.signing:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - nightly-l10n
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -185,14 +185,29 @@ implementation has its own section of th
 expects.  Thus the transforms that produce a task description must be aware of
 the worker implementation to be used, but need not be aware of the details of
 its payload format.
 
 The ``task.py`` file also contains a dictionary mapping treeherder groups to
 group names using an internal list of group names.  Feel free to add additional
 groups to this list as necessary.
 
+Signing Descriptions
+--------------------
+
+Signing kinds are passed a single dependent job (from its kind dependency) to act
+on.
+
+The transforms in ``taskcluster/taskgraph/transforms/signing.py`` implement
+this common functionality.  They expect a "signing description", and produce a
+task definition.  The schema for a signing description is defined at the top of
+``signing.py``, with copious comments.
+
+In particular you define a set of unsigned artifact urls (that point at the dependent
+task) and can optionally provide a dependent name (defaults to build) for use in
+task-reference. You also need to provide the signing format to use.
+
 More Detail
 -----------
 
 The source files provide lots of additional detail, both in the code itself and
 in the comments and docstrings.  For the next level of detail beyond this file,
 consult the transform source under ``taskcluster/taskgraph/transforms``.
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -22,18 +22,11 @@ class SigningTask(transform.TransformTas
         if (config.get('kind-dependencies', []) != ["build"] and
                 config.get('kind-dependencies', []) != ["nightly-l10n"]):
             raise Exception("Signing kinds must depend on builds")
         for task in loaded_tasks:
             if task.kind not in config.get('kind-dependencies'):
                 continue
             if not task.attributes.get('nightly'):
                 continue
-            signing_task = {}
-            signing_task['build-label'] = task.label
-            signing_task['build-platform'] = task.attributes.get('build_platform')
-            signing_task['build-type'] = task.attributes.get('build_type')
-            signing_task['build-description'] = signing_task['build-platform']
-            signing_task['build-run-on-projects'] = task.attributes.get('run_on_projects')
-            signing_task['build-treeherder'] = {}
-            if task.attributes.get('l10n_chunk'):
-                signing_task['l10n_chunk'] = task.attributes['l10n_chunk']
+            signing_task = {'dependent-task': task}
+
             yield signing_task
copy from taskcluster/taskgraph/transforms/signing.py
copy to taskcluster/taskgraph/transforms/build_signing.py
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -10,57 +10,28 @@ from __future__ import absolute_import, 
 from taskgraph.transforms.base import TransformSequence
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
 
 transforms = TransformSequence()
 
 
 @transforms.add
-def make_task_description(config, tasks):
-    for task in tasks:
-        task['label'] = task['build-label'].replace("build-", "signing-")
-        task['description'] = (task['build-description'].replace("-", " ") + " signing").title()
-        task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+def make_signing_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        job['label'] = dep_job.label.replace("build-", "signing-")
 
         artifacts = []
-        if 'android' in task['build-platform']:
+        if 'android' in dep_job.attributes.get('build_platform'):
             artifacts = ['public/build/target.apk', 'public/build/en-US/target.apk']
         else:
             artifacts = ['public/build/target.tar.bz2']
         unsigned_artifacts = []
         for artifact in artifacts:
             url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
             unsigned_artifacts.append(url)
 
-        task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
-        task['worker'] = {'implementation': 'scriptworker-signing',
-                          'unsigned-artifacts': unsigned_artifacts}
-
-        signing_format = "gpg" if "linux" in task['label'] else "jar"
-        signing_format_scope = "project:releng:signing:format:" + signing_format
-        task['scopes'] = ["project:releng:signing:cert:nightly-signing",
-                          signing_format_scope]
+        job['unsigned-artifacts'] = unsigned_artifacts
+        job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
 
-        task['dependencies'] = {'build': task['build-label']}
-        attributes = task.setdefault('attributes', {})
-        attributes['nightly'] = True
-        attributes['build_platform'] = task['build-platform']
-        attributes['build_type'] = task['build-type']
-        task['run-on-projects'] = task['build-run-on-projects']
-        task['treeherder'] = task['build-treeherder']
-        task['treeherder'].setdefault('symbol', 'tc(Ns)')
-        th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
-        th_platform = th_platform.replace("linux/opt", "linux32/opt")
-        th_platform = th_platform.replace("android-api-15/opt", "android-4-0-armv7-api15/opt")
-        task['treeherder'].setdefault('platform', th_platform)
-        task['treeherder'].setdefault('tier', 2)
-        task['treeherder'].setdefault('kind', 'build')
-
-        # delete stuff that's not part of a task description
-        del task['build-description']
-        del task['build-label']
-        del task['build-type']
-        del task['build-platform']
-        del task['build-run-on-projects']
-        del task['build-treeherder']
-
-        yield task
+        yield job
--- a/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
+++ b/taskcluster/taskgraph/transforms/nightly_l10n_signing.py
@@ -74,74 +74,48 @@ def make_pretty_name(product, build_plat
     return PRETTYNAMES[product].format(
         version=get_version_number(),
         locale=locale,
         platform=platform,
     )
 
 
 @transforms.add
-def add_signing_artifacts(config, tasks):
-    for task in tasks:
-        task['unsigned-artifacts'] = []
-        product = 'android' if 'android' in task['build-platform'] else 'desktop'
-        for locale in get_locale_list(product, task['l10n_chunk']):
-            filename = make_pretty_name(product, task['build-platform'], locale)
-            task['unsigned-artifacts'].append({
+def add_signing_artifacts(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        dep_platform = dep_job.attributes.get('build_platform')
+
+        job['unsigned-artifacts'] = []
+        product = 'android' if 'android' in dep_platform else 'desktop'
+        l10n_chunk = dep_job.attributes.get('l10n_chunk')
+        for locale in get_locale_list(product, l10n_chunk):
+            filename = make_pretty_name(product, dep_platform, locale)
+            job['unsigned-artifacts'].append({
                 'task-reference': ARTIFACT_URL.format('unsigned-repack',
                                                       filename)
                 })
             if 'tar.bz2' == filename[-7:]:
                 # Add the checksums file to be signed for linux
                 checksums_file = filename[:-7] + "checksums"
-                task['unsigned-artifacts'].append({
+                job['unsigned-artifacts'].append({
                     'task-reference': ARTIFACT_URL.format('unsigned-repack',
                                                           checksums_file)
                     })
-        yield task
+        yield job
 
 
 @transforms.add
-def make_task_description(config, tasks):
-    for task in tasks:
-        task['label'] = task['build-label'].replace("nightly-l10n-", "signing-l10n-")
-        task['description'] = (task['build-description'].replace("-", " ") +
-                               " l10n repack signing").title()
-        task['description'] = task['description'].replace("Api 15", "4.0 API15+")
-
-        unsigned_artifacts = task['unsigned-artifacts']
+def make_signing_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
 
-        task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
-        task['worker'] = {'implementation': 'scriptworker-signing',
-                          'unsigned-artifacts': unsigned_artifacts}
-
-        signing_format = "gpg" if "linux" in task['label'] else "jar"
-        signing_format_scope = "project:releng:signing:format:" + signing_format
-        task['scopes'] = ["project:releng:signing:cert:nightly-signing",
-                          signing_format_scope]
+        job['label'] = dep_job.label.replace("nightly-l10n-", "signing-l10n-")
 
-        task['dependencies'] = {'unsigned-repack': task['build-label']}
-        attributes = task.setdefault('attributes', {})
-        attributes['nightly'] = True
-        attributes['build_platform'] = task['build-platform']
-        attributes['build_type'] = task['build-type']
-        task['run-on-projects'] = task['build-run-on-projects']
-        task['treeherder'] = task['build-treeherder']
-        task['treeherder'].setdefault('symbol', 'tc(Ns{})'.format(
-            task.get('l10n_chunk', "")
-        ))
-        th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
-        th_platform = th_platform.replace("linux/opt", "linux32/opt")
-        task['treeherder'].setdefault('platform', th_platform)
-        task['treeherder'].setdefault('tier', 2)
-        task['treeherder'].setdefault('kind', 'build')
+        job['depname'] = 'unsigned-repack'
+        job['signing-format'] = "gpg" if "linux" in dep_job.label else "jar"
 
-        # delete stuff that's not part of a task description
-        del task['build-description']
-        del task['build-label']
-        del task['build-type']
-        del task['build-platform']
-        del task['build-run-on-projects']
-        del task['build-treeherder']
-        del task['l10n_chunk']
-        del task['unsigned-artifacts']
-
-        yield task
+        job['treeherder'] = {
+            # Format symbol appropriate for l10n chunking
+            'symbol': 'tc-L10n(Ns{})'.format(
+                dep_job.attributes.get('l10n_chunk')),
+        }
+        yield job
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -2,65 +2,99 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
 
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
 transforms = TransformSequence()
 
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+signing_description_schema = Schema({
+    # the dependant task (object) for this signing job, used to inform signing.
+    Required('dependent-task'): object,
+
+    # Artifacts from dep task to sign
+    Required('unsigned-artifacts'): [taskref_or_string],
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # Format to use to sign the artifacts
+    Required('signing-format'): basestring,
+
+    # unique label to describe this signing task, defaults to {dep.label}-signing
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for signing.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+})
+
 
 @transforms.add
-def make_task_description(config, tasks):
-    for task in tasks:
-        task['label'] = task['build-label'].replace("build-", "signing-")
-        task['description'] = (task['build-description'].replace("-", " ") + " signing").title()
-        task['description'] = task['description'].replace("Api 15", "4.0 API15+")
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            signing_description_schema, job,
+            "In signing ({!r} kind) task for {!r}:".format(config.kind, label))
+
 
-        artifacts = []
-        if 'android' in task['build-platform']:
-            artifacts = ['public/build/target.apk', 'public/build/en-US/target.apk']
-        else:
-            artifacts = ['public/build/target.tar.bz2']
-        unsigned_artifacts = []
-        for artifact in artifacts:
-            url = {"task-reference": ARTIFACT_URL.format('build', artifact)}
-            unsigned_artifacts.append(url)
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
 
-        task['worker-type'] = "scriptworker-prov-v1/signing-linux-v1"
-        task['worker'] = {'implementation': 'scriptworker-signing',
-                          'unsigned-artifacts': unsigned_artifacts}
+        signing_format_scope = "project:releng:signing:format:{}".format(
+            job['signing-format'])
 
-        signing_format = "gpg" if "linux" in task['label'] else "jar"
-        signing_format_scope = "project:releng:signing:format:" + signing_format
-        task['scopes'] = ["project:releng:signing:cert:nightly-signing",
-                          signing_format_scope]
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc(Ns)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        label = job.get('label', "{}-signing".format(dep_job))
 
-        task['dependencies'] = {'build': task['build-label']}
-        attributes = task.setdefault('attributes', {})
-        attributes['nightly'] = True
-        attributes['build_platform'] = task['build-platform']
-        attributes['build_type'] = task['build-type']
-        task['run-on-projects'] = task['build-run-on-projects']
-        task['treeherder'] = task['build-treeherder']
-        task['treeherder'].setdefault('symbol', 'tc(Ns)')
-        th_platform = task['build-platform'].replace("-nightly", "") + "/opt"
-        th_platform = th_platform.replace("linux/opt", "linux32/opt")
-        th_platform = th_platform.replace("android-api-15/opt", "android-4-0-armv7-api15/opt")
-        task['treeherder'].setdefault('platform', th_platform)
-        task['treeherder'].setdefault('tier', 2)
-        task['treeherder'].setdefault('kind', 'build')
-
-        # delete stuff that's not part of a task description
-        del task['build-description']
-        del task['build-label']
-        del task['build-type']
-        del task['build-platform']
-        del task['build-run-on-projects']
-        del task['build-treeherder']
+        task = {
+            'label': label,
+            'description': "{} Signing".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': "scriptworker-prov-v1/signing-linux-v1",
+            'worker': {'implementation': 'scriptworker-signing',
+                       'unsigned-artifacts': job['unsigned-artifacts']},
+            'scopes': ["project:releng:signing:cert:nightly-signing",
+                       signing_format_scope],
+            'dependencies': {job['depname']: dep_job.label},
+            'attributes': {
+                'nightly': dep_job.attributes.get('nightly', False),
+                'build_platform': dep_job.attributes.get('build_platform'),
+                'build_type': dep_job.attributes.get('build_type'),
+            },
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
 
         yield task