Bug 1362534 - Schedule windows signing (and repackage) tasks for all on-change builds r=Callek draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Tue, 27 Jun 2017 14:13:36 -0700
changeset 600969 cc13531785ecf7f55a893f744c7c143ee6255f74
parent 600694 8f80d594c08d5c7a112e5d4b9eb44ffca717eb7b
child 635143 e9ea1420cb2d6072f809bef123e0011a2e614416
push id65928
push userbmo:jlorenzo@mozilla.com
push dateTue, 27 Jun 2017 22:54:31 +0000
reviewersCallek
bugs1362534
milestone56.0a1
Bug 1362534 - Schedule windows signing (and repackage) tasks for all on-change builds r=Callek MozReview-Commit-ID: JfPTT1BH0F0
taskcluster/ci/build-signing-ci/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/push_apk.py
taskcluster/taskgraph/transforms/build_signing.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build-signing-ci/kind.yml
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file is almost identical to build-signing/kind.yml. The difference lies
+# in the attribute used. Here we don't sign nightlies, but (windows) builds for CI
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.build_signing:transforms
+   - taskgraph.transforms.signing:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build
+
+only-for-build-platforms:
+  - win32/debug
+  - win32/opt
+  - win32-pgo/opt
+  - win64/debug
+  - win64/opt
+  - win64-pgo/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -10,19 +10,26 @@ Builds are tasks that produce an install
 users or automated tests.  This is more restrictive than most definitions of
 "build" in a Mozilla context: it does not include tasks that run build-like
 actions for static analysis or to produce instrumented artifacts.
 
 build-signing
 -------------
 
 Many builds must be signed. The build-signing task takes the unsigned `build`
-kind artifacts and passess them through signingscriptworker to a signing server
+kind artifacts and passes them through signingscriptworker to a signing server
 and returns signed results.
 
+build-signing-ci
+----------------
+
+Almost identical to build-signing. The difference is: build-signing-ci is not
+meant to make shippable builds. It is only used by tests that do require signed
+binaries. 
+
 
 artifact-build
 --------------
 
 This kind performs an artifact build: one based on precompiled binaries
 discovered via the TaskCluster index.  This task verifies that such builds
 continue to work correctly.
 
--- a/taskcluster/taskgraph/loader/push_apk.py
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -9,17 +9,23 @@ from .transform import loader as base_lo
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
     """
     jobs = base_loader(kind, path, config, params, loaded_tasks)
 
     for job in jobs:
-        job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+        dependent_tasks = get_dependent_loaded_tasks(config, loaded_tasks)
+        if not dependent_tasks:
+            # PushApk must depend on signed APK. If no dependent task was found,
+            # this means another plaform (like windows) is being processed
+            continue
+
+        job['dependent-tasks'] = dependent_tasks
         yield job
 
 
 def get_dependent_loaded_tasks(config, loaded_tasks):
     nightly_tasks = (
         task for task in loaded_tasks if task.attributes.get('nightly')
     )
     tasks_with_matching_kind = (
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -6,81 +6,88 @@ Transform the signing task into an actua
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
 
+DESKTOP_BUILD_PLATFORM = ('linux', 'macosx', 'win')
+
 
 @transforms.add
 def add_signed_routes(config, jobs):
     """Add routes corresponding to the routes of the build task
        this corresponds to, with .signed inserted, for all gecko.v2 routes"""
 
     for job in jobs:
         dep_job = job['dependent-task']
 
         job['routes'] = []
-        for dep_route in dep_job.task.get('routes', []):
-            if not dep_route.startswith('index.gecko.v2'):
-                continue
-            branch = dep_route.split(".")[3]
-            rest = ".".join(dep_route.split(".")[4:])
-            job['routes'].append(
-                'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
+        if dep_job.attributes.get('nightly'):
+            for dep_route in dep_job.task.get('routes', []):
+                if not dep_route.startswith('index.gecko.v2'):
+                    continue
+                branch = dep_route.split(".")[3]
+                rest = ".".join(dep_route.split(".")[4:])
+                job['routes'].append(
+                    'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
         yield job
 
 
 @transforms.add
 def make_signing_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
-        if 'android' in dep_job.attributes.get('build_platform'):
-            job_specs = [
-                {
-                    'artifacts': ['public/build/target.apk',
-                                  'public/build/en-US/target.apk'],
-                    'format': 'jar',
-                },
-            ]
-        elif 'macosx' in dep_job.attributes.get('build_platform'):
-            job_specs = [
-                {
-                   'artifacts': ['public/build/target.dmg'],
-                   'format': 'macapp',
-                }, {
-                   'artifacts': ['public/build/update/target.complete.mar'],
-                   'format': 'mar',
-                },
-            ]
-        else:
-            job_specs = [
-                {
-                    'artifacts': ['public/build/target.tar.bz2'],
-                    'format': 'gpg',
-                }, {
-                    'artifacts': ['public/build/update/target.complete.mar'],
-                    'format': 'mar',
-                }
-            ]
-        upstream_artifacts = []
-        for spec in job_specs:
-            fmt = spec["format"]
-            upstream_artifacts.append({
-                "taskId": {"task-reference": "<build>"},
-                "taskType": "build",
-                "paths": spec["artifacts"],
-                "formats": [fmt]
-            })
-
-        job['upstream-artifacts'] = upstream_artifacts
+        job['upstream-artifacts'] = _generate_upstream_artifacts(
+            dep_job.attributes.get('build_platform'),
+            dep_job.attributes.get('nightly')
+        )
 
         label = dep_job.label.replace("build-", "signing-")
         job['label'] = label
 
         # Announce job status on funsize specific routes, so that it can
         # start the partial generation for nightlies only.
         job['use-funsize-route'] = True
 
         yield job
+
+
+def _generate_upstream_artifacts(build_platform, is_nightly=False):
+    if 'android' in build_platform:
+        artifacts_specificities = [{
+            'artifacts': [
+                'public/build/target.apk',
+                'public/build/en-US/target.apk'
+            ],
+            'format': 'jar',
+        }]
+    elif 'macosx' in build_platform:
+        artifacts_specificities = [{
+            'artifacts': ['public/build/target.dmg'],
+            'format': 'macapp',
+        }]
+    elif 'win' in build_platform:
+        artifacts_specificities = [{
+            'artifacts': ['public/build/target.zip'],
+            'format': 'sha2signcode',
+        }]
+    else:
+        artifacts_specificities = [{
+            'artifacts': ['public/build/target.tar.bz2'],
+            'format': 'gpg',
+        }]
+
+    if is_nightly and any(desktop in build_platform for desktop in DESKTOP_BUILD_PLATFORM):
+        artifacts_specificities.append({
+            'artifacts': ['public/build/update/target.complete.mar'],
+            'format': 'mar',
+        })
+
+    return [{
+        'taskId': {'task-reference': '<build>'},
+        'taskType': 'build',
+        'paths': specificity['artifacts'],
+        'formats': [specificity['format']],
+    } for specificity in artifacts_specificities]
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -4,17 +4,18 @@
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope, get_devedition_signing_cert_scope
+from taskgraph.util.scriptworker import get_signing_cert_scope, get_ci_signing_cert_scope, \
+    get_devedition_signing_cert_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -83,43 +84,53 @@ def make_task_description(config, jobs):
         formats = set([])
         for artifacts in job['upstream-artifacts']:
             for f in artifacts['formats']:
                 formats.add(f)  # Add each format only once
         for format in formats:
             signing_format_scopes.append("project:releng:signing:format:{}".format(format))
 
         treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'tc(Ns)')
+        is_nightly = dep_job.attributes.get('nightly', False)
+        treeherder.setdefault('symbol', 'tc(Ns)' if is_nightly else 'tc(Bs)')
+
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
-        treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
-        treeherder.setdefault('tier', 1)
+        build_type = dep_job.attributes.get('build_type')
+        build_platform = dep_job.attributes.get('build_platform')
+        treeherder.setdefault('platform', _generate_treeherder_platform(
+            dep_th_platform, build_platform, build_type
+        ))
+
+        # TODO: Make non-nightly (aka build-signing-ci) Tier 1 once mature enough
+        treeherder.setdefault('tier', 1 if is_nightly else 3)
         treeherder.setdefault('kind', 'build')
 
         label = job.get('label', "{}-signing".format(dep_job.label))
 
         attributes = {
-            'nightly': dep_job.attributes.get('nightly', False),
-            'build_platform': dep_job.attributes.get('build_platform'),
-            'build_type': dep_job.attributes.get('build_type'),
+            'nightly': is_nightly,
+            'build_platform': build_platform,
+            'build_type': build_type,
             'signed': True,
         }
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
         # This code wasn't originally written with the possibility of using different
         # signing cert scopes for different platforms on the same branch. This isn't
         # ideal, but it's what we currently have to make this possible.
         if dep_job.attributes.get('build_platform') in set(
           ['linux64-devedition-nightly', 'linux-devedition-nightly']):
             signing_cert_scope = get_devedition_signing_cert_scope(config)
+        elif is_nightly:
+            signing_cert_scope = get_signing_cert_scope(config)
         else:
-            signing_cert_scope = get_signing_cert_scope(config)
+            signing_cert_scope = get_ci_signing_cert_scope(config)
 
         task = {
             'label': label,
             'description': "{} Signing".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
@@ -133,8 +144,13 @@ def make_task_description(config, jobs):
         }
 
         if 'macosx' not in dep_job.attributes.get('build_platform') and \
                 job.get('use-funsize-route', False):
             task['routes'].append("project.releng.funsize.level-{level}.{project}".format(
                 project=config.params['project'], level=config.params['level']))
 
         yield task
+
+
+def _generate_treeherder_platform(dep_th_platform, build_platform, build_type):
+    actual_build_type = 'pgo' if '-pgo' in build_platform else build_type
+    return '{}/{}'.format(dep_th_platform, actual_build_type)
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -51,16 +51,28 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
 """Map the signing scope aliases to the actual scopes.
 """
 SIGNING_CERT_SCOPES = {
     'all-release-branches': 'project:releng:signing:cert:release-signing',
     'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
     'default': 'project:releng:signing:cert:dep-signing',
 }
 
+CI_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
+    'central', set([
+        'mozilla-central',
+    ])
+]]
+
+CI_SIGNING_CERT_SCOPES = {
+    # No branch-specific configuration required at the moment
+    'central': 'project:releng:signing:cert:dep-signing',
+    'default': 'project:releng:signing:cert:dep-signing',
+}
+
 DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
     'beta', set([
         'mozilla-beta',
     ])
 ]]
 
 DEVEDITION_SIGNING_CERT_SCOPES = {
     'beta': 'project:releng:signing:cert:nightly-signing',
@@ -323,16 +335,22 @@ def get_scope_from_target_method_and_pro
 
 
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     SIGNING_SCOPE_ALIAS_TO_PROJECT,
     SIGNING_CERT_SCOPES
 )
 
+get_ci_signing_cert_scope = functools.partial(
+    get_scope_from_project,
+    CI_SIGNING_SCOPE_ALIAS_TO_PROJECT,
+    CI_SIGNING_CERT_SCOPES
+)
+
 get_devedition_signing_cert_scope = functools.partial(
     get_scope_from_project,
     DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
     DEVEDITION_SIGNING_CERT_SCOPES
 )
 
 get_beetmover_bucket_scope = functools.partial(
     get_scope_from_target_method_and_project,