Bug 1362534 - Schedule windows signing (and repackage) tasks for all on-change builds r=Callek
MozReview-Commit-ID: JfPTT1BH0F0
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/build-signing-ci/kind.yml
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file is almost identical to build-signing/kind.yml. The difference lies
+# in the attribute used. Here we don't sign nightlies, but (windows) builds for CI
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+ - taskgraph.transforms.build_signing:transforms
+ - taskgraph.transforms.signing:transforms
+ - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+ - build
+
+only-for-build-platforms:
+ - win32/debug
+ - win32/opt
+ - win32-pgo/opt
+ - win64/debug
+ - win64/opt
+ - win64-pgo/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -10,19 +10,26 @@ Builds are tasks that produce an install
users or automated tests. This is more restrictive than most definitions of
"build" in a Mozilla context: it does not include tasks that run build-like
actions for static analysis or to produce instrumented artifacts.
build-signing
-------------
Many builds must be signed. The build-signing task takes the unsigned `build`
-kind artifacts and passess them through signingscriptworker to a signing server
+kind artifacts and passes them through signingscriptworker to a signing server
and returns signed results.
+build-signing-ci
+----------------
+
+Almost identical to build-signing. The difference is: build-signing-ci is not
+meant to make shippable builds. It is only used by tests that do require signed
+binaries.
+
artifact-build
--------------
This kind performs an artifact build: one based on precompiled binaries
discovered via the TaskCluster index. This task verifies that such builds
continue to work correctly.
--- a/taskcluster/taskgraph/loader/push_apk.py
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -9,17 +9,23 @@ from .transform import loader as base_lo
def loader(kind, path, config, params, loaded_tasks):
"""
Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
"""
jobs = base_loader(kind, path, config, params, loaded_tasks)
for job in jobs:
- job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+ dependent_tasks = get_dependent_loaded_tasks(config, loaded_tasks)
+ if not dependent_tasks:
+ # PushApk must depend on signed APK. If no dependent task was found,
+ # this means another plaform (like windows) is being processed
+ continue
+
+ job['dependent-tasks'] = dependent_tasks
yield job
def get_dependent_loaded_tasks(config, loaded_tasks):
nightly_tasks = (
task for task in loaded_tasks if task.attributes.get('nightly')
)
tasks_with_matching_kind = (
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -6,81 +6,88 @@ Transform the signing task into an actua
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
+DESKTOP_BUILD_PLATFORM = ('linux', 'macosx', 'win')
+
@transforms.add
def add_signed_routes(config, jobs):
"""Add routes corresponding to the routes of the build task
this corresponds to, with .signed inserted, for all gecko.v2 routes"""
for job in jobs:
dep_job = job['dependent-task']
job['routes'] = []
- for dep_route in dep_job.task.get('routes', []):
- if not dep_route.startswith('index.gecko.v2'):
- continue
- branch = dep_route.split(".")[3]
- rest = ".".join(dep_route.split(".")[4:])
- job['routes'].append(
- 'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
+ if dep_job.attributes.get('nightly'):
+ for dep_route in dep_job.task.get('routes', []):
+ if not dep_route.startswith('index.gecko.v2'):
+ continue
+ branch = dep_route.split(".")[3]
+ rest = ".".join(dep_route.split(".")[4:])
+ job['routes'].append(
+ 'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
yield job
@transforms.add
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
- if 'android' in dep_job.attributes.get('build_platform'):
- job_specs = [
- {
- 'artifacts': ['public/build/target.apk',
- 'public/build/en-US/target.apk'],
- 'format': 'jar',
- },
- ]
- elif 'macosx' in dep_job.attributes.get('build_platform'):
- job_specs = [
- {
- 'artifacts': ['public/build/target.dmg'],
- 'format': 'macapp',
- }, {
- 'artifacts': ['public/build/update/target.complete.mar'],
- 'format': 'mar',
- },
- ]
- else:
- job_specs = [
- {
- 'artifacts': ['public/build/target.tar.bz2'],
- 'format': 'gpg',
- }, {
- 'artifacts': ['public/build/update/target.complete.mar'],
- 'format': 'mar',
- }
- ]
- upstream_artifacts = []
- for spec in job_specs:
- fmt = spec["format"]
- upstream_artifacts.append({
- "taskId": {"task-reference": "<build>"},
- "taskType": "build",
- "paths": spec["artifacts"],
- "formats": [fmt]
- })
-
- job['upstream-artifacts'] = upstream_artifacts
+ job['upstream-artifacts'] = _generate_upstream_artifacts(
+ dep_job.attributes.get('build_platform'),
+ dep_job.attributes.get('nightly')
+ )
label = dep_job.label.replace("build-", "signing-")
job['label'] = label
# Announce job status on funsize specific routes, so that it can
# start the partial generation for nightlies only.
job['use-funsize-route'] = True
yield job
+
+
+def _generate_upstream_artifacts(build_platform, is_nightly=False):
+ if 'android' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': [
+ 'public/build/target.apk',
+ 'public/build/en-US/target.apk'
+ ],
+ 'format': 'jar',
+ }]
+ elif 'macosx' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.dmg'],
+ 'format': 'macapp',
+ }]
+ elif 'win' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.zip'],
+ 'format': 'sha2signcode',
+ }]
+ else:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.tar.bz2'],
+ 'format': 'gpg',
+ }]
+
+ if is_nightly and any(desktop in build_platform for desktop in DESKTOP_BUILD_PLATFORM):
+ artifacts_specificities.append({
+ 'artifacts': ['public/build/update/target.complete.mar'],
+ 'format': 'mar',
+ })
+
+ return [{
+ 'taskId': {'task-reference': '<build>'},
+ 'taskType': 'build',
+ 'paths': specificity['artifacts'],
+ 'formats': [specificity['format']],
+ } for specificity in artifacts_specificities]
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -4,17 +4,18 @@
"""
Transform the signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope, get_devedition_signing_cert_scope
+from taskgraph.util.scriptworker import get_signing_cert_scope, get_ci_signing_cert_scope, \
+ get_devedition_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -83,43 +84,53 @@ def make_task_description(config, jobs):
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append("project:releng:signing:format:{}".format(format))
treeherder = job.get('treeherder', {})
- treeherder.setdefault('symbol', 'tc(Ns)')
+ is_nightly = dep_job.attributes.get('nightly', False)
+ treeherder.setdefault('symbol', 'tc(Ns)' if is_nightly else 'tc(Bs)')
+
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
- treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
- treeherder.setdefault('tier', 1)
+ build_type = dep_job.attributes.get('build_type')
+ build_platform = dep_job.attributes.get('build_platform')
+ treeherder.setdefault('platform', _generate_treeherder_platform(
+ dep_th_platform, build_platform, build_type
+ ))
+
+ # TODO: Make non-nightly (aka build-signing-ci) Tier 1 once mature enough
+ treeherder.setdefault('tier', 1 if is_nightly else 3)
treeherder.setdefault('kind', 'build')
label = job.get('label', "{}-signing".format(dep_job.label))
attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
+ 'nightly': is_nightly,
+ 'build_platform': build_platform,
+ 'build_type': build_type,
'signed': True,
}
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
# This code wasn't originally written with the possibility of using different
# signing cert scopes for different platforms on the same branch. This isn't
# ideal, but it's what we currently have to make this possible.
if dep_job.attributes.get('build_platform') in set(
['linux64-devedition-nightly', 'linux-devedition-nightly']):
signing_cert_scope = get_devedition_signing_cert_scope(config)
+ elif is_nightly:
+ signing_cert_scope = get_signing_cert_scope(config)
else:
- signing_cert_scope = get_signing_cert_scope(config)
+ signing_cert_scope = get_ci_signing_cert_scope(config)
task = {
'label': label,
'description': "{} Signing".format(
dep_job.task["metadata"]["description"]),
'worker-type': "scriptworker-prov-v1/signing-linux-v1",
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
@@ -133,8 +144,13 @@ def make_task_description(config, jobs):
}
if 'macosx' not in dep_job.attributes.get('build_platform') and \
job.get('use-funsize-route', False):
task['routes'].append("project.releng.funsize.level-{level}.{project}".format(
project=config.params['project'], level=config.params['level']))
yield task
+
+
+def _generate_treeherder_platform(dep_th_platform, build_platform, build_type):
+ actual_build_type = 'pgo' if '-pgo' in build_platform else build_type
+ return '{}/{}'.format(dep_th_platform, actual_build_type)
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -51,16 +51,28 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
"""Map the signing scope aliases to the actual scopes.
"""
SIGNING_CERT_SCOPES = {
'all-release-branches': 'project:releng:signing:cert:release-signing',
'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
'default': 'project:releng:signing:cert:dep-signing',
}
+CI_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
+ 'central', set([
+ 'mozilla-central',
+ ])
+]]
+
+CI_SIGNING_CERT_SCOPES = {
+ # No branch-specific configuration required at the moment
+ 'central': 'project:releng:signing:cert:dep-signing',
+ 'default': 'project:releng:signing:cert:dep-signing',
+}
+
DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
'beta', set([
'mozilla-beta',
])
]]
DEVEDITION_SIGNING_CERT_SCOPES = {
'beta': 'project:releng:signing:cert:nightly-signing',
@@ -323,16 +335,22 @@ def get_scope_from_target_method_and_pro
get_signing_cert_scope = functools.partial(
get_scope_from_project,
SIGNING_SCOPE_ALIAS_TO_PROJECT,
SIGNING_CERT_SCOPES
)
+get_ci_signing_cert_scope = functools.partial(
+ get_scope_from_project,
+ CI_SIGNING_SCOPE_ALIAS_TO_PROJECT,
+ CI_SIGNING_CERT_SCOPES
+)
+
get_devedition_signing_cert_scope = functools.partial(
get_scope_from_project,
DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
DEVEDITION_SIGNING_CERT_SCOPES
)
get_beetmover_bucket_scope = functools.partial(
get_scope_from_target_method_and_project,