--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@@ -1,19 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-loader: taskgraph.loader.single_dep:loader
+loader: taskgraph.loader.build_signing:loader
transforms:
- taskgraph.transforms.build_signing:transforms
- taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
-only-for-attributes:
- - nightly
-
not-for-build-platforms:
- win64-nightly/opt
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -1,11 +1,12 @@
loader: taskgraph.loader.test:loader
kind-dependencies:
- build
+ - build-signing
transforms:
- taskgraph.transforms.tests:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
parse-commit: taskgraph.try_option_syntax:parse_message
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -118,17 +118,17 @@ windows-vm-tests:
- mochitest
- mochitest-a11y
- mochitest-browser-chrome
- mochitest-devtools-chrome
- mochitest-jetpack
- mochitest-media
- web-platform-tests
- web-platform-tests-reftests
- #- xpcshell
+ - xpcshell
windows-gpu-tests:
- reftest
- reftest-no-accel
- mochitest-gpu
- mochitest-webgl
# these tests currently run on hardware, but may migrate above when validated
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -1565,32 +1565,37 @@ xpcshell:
description: "xpcshell test run"
suite:
by-test-platform:
linux64-jsdcov/opt: xpcshell-coverage
default: xpcshell
treeherder-symbol: tc-X(X)
run-on-projects:
by-test-platform:
- windows.*: []
linux64-qr/.*: ['mozilla-central', 'try']
+ # TODO Windows: Turn on on the other branches once Bug 1374787 is fixed
+ windows.*: ['mozilla-central', 'try']
default: built-projects
docker-image: {"in-tree": "desktop1604-test"}
chunks:
by-test-platform:
linux64/debug: 10
android-4.2-x86/opt: 6
- # windows.*: 1
macosx.*: 1
+ windows.*: 1
default: 8
instance-size:
by-test-platform:
android.*: xlarge
default: legacy # Bug 1281241: migrating to m3.large instances
max-run-time: 5400
+ tier:
+ by-test-platform:
+ windows.*: 3
+ default: default
e10s: false
allow-software-gl-layers: false
mozharness:
by-test-platform:
android.*:
script: android_emulator_unittest.py
no-read-buildbot-config: true
extra-options:
@@ -1614,8 +1619,13 @@ xpcshell:
- unittests/linux_unittest.py
- remove_executables.py
extra-options:
by-test-platform:
linux64-jsdcov/opt:
- --xpcshell-suite=xpcshell-coverage
default:
- --xpcshell-suite=xpcshell
+ requires-signed-builds:
+ by-test-platform:
+ windows10-64-asan/opt: false # No XPCShell on ASAN yet
+ windows.*: true
+ default: false
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -10,20 +10,19 @@ Builds are tasks that produce an install
users or automated tests. This is more restrictive than most definitions of
"build" in a Mozilla context: it does not include tasks that run build-like
actions for static analysis or to produce instrumented artifacts.
build-signing
-------------
Many builds must be signed. The build-signing task takes the unsigned `build`
-kind artifacts and passess them through signingscriptworker to a signing server
+kind artifacts and passes them through signingscriptworker to a signing server
and returns signed results.
-
artifact-build
--------------
This kind performs an artifact build: one based on precompiled binaries
discovered via the TaskCluster index. This task verifies that such builds
continue to work correctly.
hazard
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/build_signing.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.loader.single_dep import loader as base_loader
+
+# XXX: This logic should rely in kind.yml. This hasn't been done in the original
+# patch because it required some heavy changes in single_dep.
+LABELS_WHICH_SHOULD_SIGN_CI_BUILDS = (
+ 'build-win32/debug', 'build-win32/opt', 'build-win32-pgo/opt',
+ 'build-win64/debug', 'build-win64/opt', 'build-win64-pgo/opt',
+)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+ jobs = base_loader(kind, path, config, params, loaded_tasks)
+
+ for job in jobs:
+ dependent_task = job['dependent-task']
+ if dependent_task.attributes.get('nightly') or \
+ dependent_task.label in LABELS_WHICH_SHOULD_SIGN_CI_BUILDS:
+ yield job
--- a/taskcluster/taskgraph/loader/push_apk.py
+++ b/taskcluster/taskgraph/loader/push_apk.py
@@ -9,17 +9,23 @@ from .transform import loader as base_lo
def loader(kind, path, config, params, loaded_tasks):
"""
Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
"""
jobs = base_loader(kind, path, config, params, loaded_tasks)
for job in jobs:
- job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+ dependent_tasks = get_dependent_loaded_tasks(config, loaded_tasks)
+ if not dependent_tasks:
+ # PushApk must depend on signed APK. If no dependent task was found,
+ # this means another plaform (like windows) is being processed
+ continue
+
+ job['dependent-tasks'] = dependent_tasks
yield job
def get_dependent_loaded_tasks(config, loaded_tasks):
nightly_tasks = (
task for task in loaded_tasks if task.attributes.get('nightly')
)
tasks_with_matching_kind = (
--- a/taskcluster/taskgraph/loader/test.py
+++ b/taskcluster/taskgraph/loader/test.py
@@ -13,45 +13,52 @@ logger = logging.getLogger(__name__)
def loader(kind, path, config, params, loaded_tasks):
"""
Generate tasks implementing Gecko tests.
"""
# the kind on which this one depends
- if len(config.get('kind-dependencies', [])) != 1:
+ if len(config.get('kind-dependencies', [])) != 2:
raise Exception(
- "Test kinds must have exactly one item in kind-dependencies")
- dep_kind = config['kind-dependencies'][0]
+ 'Test kinds must have exactly 2 items in kind-dependencies'
+ )
- # get build tasks, keyed by build platform
- builds_by_platform = get_builds_by_platform(dep_kind, loaded_tasks)
+ builds_by_platform = get_builds_by_platform(dep_kind='build', loaded_tasks=loaded_tasks)
+ signed_builds_by_platform = get_builds_by_platform(
+ dep_kind='build-signing', loaded_tasks=loaded_tasks
+ )
# get the test platforms for those build tasks
test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
- test_platforms = get_test_platforms(test_platforms_cfg, builds_by_platform)
+ test_platforms = get_test_platforms(
+ test_platforms_cfg, builds_by_platform, signed_builds_by_platform
+ )
# expand the test sets for each of those platforms
test_sets_cfg = load_yaml(path, 'test-sets.yml')
test_platforms = expand_tests(test_sets_cfg, test_platforms)
# load the test descriptions
test_descriptions = load_yaml(path, 'tests.yml', enforce_order=True)
# generate all tests for all test platforms
for test_platform_name, test_platform in test_platforms.iteritems():
for test_name in test_platform['test-names']:
test = copy.deepcopy(test_descriptions[test_name])
test['build-platform'] = test_platform['build-platform']
test['test-platform'] = test_platform_name
test['build-label'] = test_platform['build-label']
+ if test_platform.get('build-signing-label', None):
+ test['build-signing-label'] = test_platform['build-signing-label']
+
test['build-attributes'] = test_platform['build-attributes']
test['test-name'] = test_name
- if test_platform['nightly']:
+ if test_platform.get('nightly'):
test.setdefault('attributes', {})['nightly'] = True
logger.debug("Generating tasks for test {} on platform {}".format(
test_name, test['test-platform']))
yield test
def get_builds_by_platform(dep_kind, loaded_tasks):
@@ -68,35 +75,45 @@ def get_builds_by_platform(dep_kind, loa
continue
platform = "{}/{}".format(build_platform, build_type)
if platform in builds_by_platform:
raise Exception("multiple build jobs for " + platform)
builds_by_platform[platform] = task
return builds_by_platform
-def get_test_platforms(test_platforms_cfg, builds_by_platform):
+def get_test_platforms(test_platforms_cfg, builds_by_platform, signed_builds_by_platform={}):
"""Get the test platforms for which test tasks should be generated,
based on the available build platforms. Returns a dictionary mapping
test platform to {test-set, build-platform, build-label}."""
test_platforms = {}
for test_platform, cfg in test_platforms_cfg.iteritems():
build_platform = cfg['build-platform']
if build_platform not in builds_by_platform:
logger.warning(
"No build task with platform {}; ignoring test platform {}".format(
build_platform, test_platform))
continue
test_platforms[test_platform] = {
- 'nightly': builds_by_platform[build_platform].attributes.get('nightly', False),
'build-platform': build_platform,
'build-label': builds_by_platform[build_platform].label,
'build-attributes': builds_by_platform[build_platform].attributes,
}
+
+ if builds_by_platform[build_platform].attributes.get('nightly'):
+ test_platforms[test_platform]['nightly'] = \
+ builds_by_platform[build_platform].attributes['nightly']
+
test_platforms[test_platform].update(cfg)
+
+ if build_platform in signed_builds_by_platform:
+ # Context: Signed builds are only used by Windows
+ test_platforms[test_platform]['build-signing-label'] = \
+ signed_builds_by_platform[build_platform].label
+
return test_platforms
def expand_tests(test_sets_cfg, test_platforms):
"""Expand the test sets in `test_platforms` out to sets of test names.
Returns a dictionary like `get_test_platforms`, with an additional
`test-names` key for each test platform, containing a set of test
names."""
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_balrog_server_scope,
get_balrog_channel_scopes)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -58,21 +59,17 @@ def make_task_description(config, jobs):
treeherder.setdefault('symbol', 'tc-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
label = job.get('label', "balrog-{}".format(dep_job.label))
upstream_artifacts = [{
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
@@ -263,22 +264,18 @@ def make_task_description(config, jobs):
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'signed': dep_job.attributes.get('signed', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
@@ -58,22 +59,18 @@ def make_beetmover_checksums_description
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'signed': dep_job.attributes.get('signed', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
import logging
logger = logging.getLogger(__name__)
@@ -161,22 +162,17 @@ def make_task_description(config, jobs):
repackage_name = "repackage"
# repackage-l10n actually uses the repackage depname here
repackage_dependencies = {"repackage":
dep_job.dependencies[repackage_name]
}
dependencies.update(repackage_dependencies)
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'signed': dep_job.attributes.get('signed', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
--- a/taskcluster/taskgraph/transforms/build.py
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -17,28 +17,32 @@ transforms = TransformSequence()
@transforms.add
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job['treeherder'].setdefault('kind', 'build')
job['treeherder'].setdefault('tier', 1)
job.setdefault('needs-sccache', True)
_, worker_os = worker_type_implementation(job['worker-type'])
+ worker = job.setdefault('worker', {})
if worker_os == "linux":
- worker = job.setdefault('worker')
worker.setdefault('docker-image', {'in-tree': 'desktop-build'})
worker['chain-of-trust'] = True
extra = job.setdefault('extra', {})
extra.setdefault('chainOfTrust', {})
extra['chainOfTrust'].setdefault('inputs', {})
extra['chainOfTrust']['inputs']['docker-image'] = {
"task-reference": "<docker-image>"
}
- elif worker_os in set(["macosx", "windows"]):
- job['worker'].setdefault('env', {})
+ elif worker_os == "windows":
+ worker.setdefault('env', {})
+ worker['chain-of-trust'] = True
+ elif worker_os == "macosx":
+ worker.setdefault('env', {})
+
yield job
@transforms.add
def set_env(config, jobs):
"""Set extra environment variables from try command line."""
for job in jobs:
env = config.config['args'].env
--- a/taskcluster/taskgraph/transforms/build_signing.py
+++ b/taskcluster/taskgraph/transforms/build_signing.py
@@ -16,68 +16,76 @@ transforms = TransformSequence()
def add_signed_routes(config, jobs):
"""Add routes corresponding to the routes of the build task
this corresponds to, with .signed inserted, for all gecko.v2 routes"""
for job in jobs:
dep_job = job['dependent-task']
job['routes'] = []
- for dep_route in dep_job.task.get('routes', []):
- if not dep_route.startswith('index.gecko.v2'):
- continue
- branch = dep_route.split(".")[3]
- rest = ".".join(dep_route.split(".")[4:])
- job['routes'].append(
- 'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
+ if dep_job.attributes.get('nightly'):
+ for dep_route in dep_job.task.get('routes', []):
+ if not dep_route.startswith('index.gecko.v2'):
+ continue
+ branch = dep_route.split(".")[3]
+ rest = ".".join(dep_route.split(".")[4:])
+ job['routes'].append(
+ 'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
+
yield job
@transforms.add
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
- if 'android' in dep_job.attributes.get('build_platform'):
- job_specs = [
- {
- 'artifacts': ['public/build/target.apk',
- 'public/build/en-US/target.apk'],
- 'format': 'jar',
- },
- ]
- elif 'macosx' in dep_job.attributes.get('build_platform'):
- job_specs = [
- {
- 'artifacts': ['public/build/target.dmg'],
- 'format': 'macapp',
- },
- ]
- else:
- job_specs = [
- {
- 'artifacts': ['public/build/target.tar.bz2'],
- 'format': 'gpg',
- }, {
- 'artifacts': ['public/build/update/target.complete.mar'],
- 'format': 'mar',
- }
- ]
- upstream_artifacts = []
- for spec in job_specs:
- fmt = spec["format"]
- upstream_artifacts.append({
- "taskId": {"task-reference": "<build>"},
- "taskType": "build",
- "paths": spec["artifacts"],
- "formats": [fmt]
- })
-
- job['upstream-artifacts'] = upstream_artifacts
+ job['upstream-artifacts'] = _generate_upstream_artifacts(
+ dep_job.attributes.get('build_platform'),
+ dep_job.attributes.get('nightly')
+ )
label = dep_job.label.replace("build-", "signing-")
job['label'] = label
# Announce job status on funsize specific routes, so that it can
# start the partial generation for nightlies only.
job['use-funsize-route'] = True
yield job
+
+
+def _generate_upstream_artifacts(build_platform, is_nightly=False):
+ if 'android' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': [
+ 'public/build/target.apk',
+ 'public/build/en-US/target.apk'
+ ],
+ 'format': 'jar',
+ }]
+ # XXX: Mac and Windows don't sign mars because internal aren't signed at
+ # this stage of the release
+ elif 'macosx' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.dmg'],
+ 'format': 'macapp',
+ }]
+ elif 'win' in build_platform:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.zip'],
+ 'format': 'sha2signcode',
+ }]
+ else:
+ artifacts_specificities = [{
+ 'artifacts': ['public/build/target.tar.bz2'],
+ 'format': 'gpg',
+ }, {
+ 'artifacts': ['public/build/update/target.complete.mar'],
+ 'format': 'mar',
+ }]
+
+ return [{
+ 'taskId': {'task-reference': '<build>'},
+ 'taskType': 'build',
+ 'paths': specificity['artifacts'],
+ 'formats': [specificity['format']],
+ } for specificity in artifacts_specificities]
--- a/taskcluster/taskgraph/transforms/checksums_signing.py
+++ b/taskcluster/taskgraph/transforms/checksums_signing.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the checksums signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import get_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -52,21 +53,18 @@ def make_checksums_signing_description(c
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "checksumssigning-{}".format(dep_job.label))
dependencies = {"beetmover": dep_job.label}
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-cs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {"task-reference": "<beetmover>"},
"taskType": "beetmover",
"paths": [
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -76,16 +76,18 @@ mozharness_run_schema = Schema({
Required('need-xvfb', default=False): bool,
# If false, indicate that builds should skip producing artifacts. Not
# supported on Windows.
Required('keep-artifacts', default=True): bool,
# If specified, use the in-tree job script specified.
Optional('job-script'): basestring,
+
+ Required('requires-signed-builds', default=False): bool,
})
@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -198,17 +198,18 @@ def mozharness_test_on_generic_worker(co
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
- installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
+ upstream_task = '<build-signing>' if mozharness['requires-signed-builds'] else '<build>'
+ installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name'])
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot']))
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the repackage task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
@@ -104,22 +105,19 @@ def make_job_description(config, jobs):
# We need this additional dependency to support finding the mar binary
# Which is needed in order to generate a new complete.mar
dependencies['build'] = "build-{}/opt".format(
dependencies[build_task][13:dependencies[build_task].rfind('-')])
build_task = 'build'
signing_task_ref = "<{}>".format(signing_task)
build_task_ref = "<{}>".format(build_task)
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- 'repackage_type': 'repackage',
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+ attributes['repackage_type'] = 'repackage'
+
if job.get('locale'):
attributes['locale'] = job['locale']
level = config.params['level']
task_env = {}
locale_output_path = ""
mar_prefix = 'https://queue.taskcluster.net/v1/task/' + \
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the repackage signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import get_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -52,22 +53,19 @@ def make_repackage_signing_description(c
label = job.get('label', "repackage-signing-{}".format(dep_job.label))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- 'repackage_type': 'repackage-signing',
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+ attributes['repackage_type'] = 'repackage-signing'
+
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-rs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
scopes = [get_signing_cert_scope(config),
"project:releng:signing:format:mar"]
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the signing task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import get_signing_cert_scope, get_devedition_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
@@ -83,49 +84,57 @@ def make_task_description(config, jobs):
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append("project:releng:signing:format:{}".format(format))
treeherder = job.get('treeherder', {})
- treeherder.setdefault('symbol', 'tc(Ns)')
+ is_nightly = dep_job.attributes.get('nightly', False)
+ treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly))
+
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
- treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
- treeherder.setdefault('tier', 1)
+ build_type = dep_job.attributes.get('build_type')
+ build_platform = dep_job.attributes.get('build_platform')
+ treeherder.setdefault('platform', _generate_treeherder_platform(
+ dep_th_platform, build_platform, build_type
+ ))
+
+ # TODO: Make non-nightly (i.e. windows CI builds) Tier 1 once green on
+ # central, inbound, autoland and try
+ treeherder.setdefault('tier', 1 if is_nightly else 3)
treeherder.setdefault('kind', 'build')
label = job.get('label', "{}-signing".format(dep_job.label))
- attributes = {
- 'nightly': dep_job.attributes.get('nightly', False),
- 'build_platform': dep_job.attributes.get('build_platform'),
- 'build_type': dep_job.attributes.get('build_type'),
- 'signed': True,
- }
+ attributes = copy_attributes_from_dependent_job(dep_job)
+ attributes['signed'] = True
+
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
# This code wasn't originally written with the possibility of using different
# signing cert scopes for different platforms on the same branch. This isn't
# ideal, but it's what we currently have to make this possible.
if dep_job.attributes.get('build_platform') in set(
['linux64-devedition-nightly', 'linux-devedition-nightly']):
signing_cert_scope = get_devedition_signing_cert_scope(config)
+ elif is_nightly:
+ signing_cert_scope = get_signing_cert_scope(config)
else:
- signing_cert_scope = get_signing_cert_scope(config)
+ signing_cert_scope = 'project:releng:signing:cert:dep-signing'
task = {
'label': label,
'description': "{} Signing".format(
dep_job.task["metadata"]["description"]),
- 'worker-type': "scriptworker-prov-v1/signing-linux-v1",
+ 'worker-type': _generate_worker_type(signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
@@ -133,8 +142,22 @@ def make_task_description(config, jobs):
}
if 'macosx' not in dep_job.attributes.get('build_platform') and \
job.get('use-funsize-route', False):
task['routes'].append("project.releng.funsize.level-{level}.{project}".format(
project=config.params['project'], level=config.params['level']))
yield task
+
+
+def _generate_treeherder_platform(dep_th_platform, build_platform, build_type):
+ actual_build_type = 'pgo' if '-pgo' in build_platform else build_type
+ return '{}/{}'.format(dep_th_platform, actual_build_type)
+
+
+def _generate_treeherder_symbol(is_nightly):
+ return 'tc(Ns)' if is_nightly else 'tc(Bs)'
+
+
+def _generate_worker_type(signing_cert_scope):
+ worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
+ return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -244,16 +244,20 @@ test_description_schema = Schema({
# Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below
'test-suite-suffix',
),
# the string to append to the `--test-suite` arugment when
# chunking-args = test-suite-suffix; "<CHUNK>" in this string will
# be replaced with the chunk number.
Optional('chunk-suffix'): basestring,
+
+ Required('requires-signed-builds', default=False): optionally_keyed_by(
+ 'test-platform',
+ bool),
}
),
# The current chunk; this is filled in by `all_kinds.py`
Optional('this-chunk'): int,
# os user groups for test task workers; required scopes, will be
# added automatically
@@ -264,16 +268,20 @@ test_description_schema = Schema({
# -- values supplied by the task-generation infrastructure
# the platform of the build this task is testing
'build-platform': basestring,
# the label of the build task generating the materials to test
'build-label': basestring,
+ # the label of the signing task generating the materials to test.
+ # Signed builds are used in xpcshell tests on Windows, for instance.
+ Optional('build-signing-label'): basestring,
+
# the build's attributes
'build-attributes': {basestring: object},
# the platform on which the tests will run
'test-platform': basestring,
# the name of the test (the key in tests.yml)
'test-name': basestring,
@@ -382,16 +390,17 @@ def set_target(config, tests):
target = 'geckoview_example.apk'
else:
target = 'target.apk'
elif build_platform.startswith('win'):
target = 'target.zip'
else:
target = 'target.tar.bz2'
test['mozharness']['build-artifact-name'] = 'public/build/' + target
+
yield test
@transforms.add
def set_treeherder_machine_platform(config, tests):
"""Set the appropriate task.extra.treeherder.machine.platform"""
translation = {
# Linux64 build platforms for asan and pgo are specified differently to
@@ -487,16 +496,17 @@ def handle_keyed_by(config, tests):
'chunks',
'e10s',
'suite',
'run-on-projects',
'os-groups',
'mozharness.chunked',
'mozharness.config',
'mozharness.extra-options',
+ 'mozharness.requires-signed-builds',
]
for test in tests:
for field in fields:
resolve_keyed_by(test, field, item_name=test['test-name'],
project=config.params['project'])
yield test
@@ -770,16 +780,20 @@ def make_job_description(config, tests):
jobdesc = {}
name = '{}-{}'.format(test['test-platform'], test['test-name'])
jobdesc['name'] = name
jobdesc['label'] = label
jobdesc['description'] = test['description']
jobdesc['when'] = test.get('when', {})
jobdesc['attributes'] = attributes
jobdesc['dependencies'] = {'build': build_label}
+
+ if test['mozharness']['requires-signed-builds'] is True:
+ jobdesc['dependencies']['build-signing'] = test['build-signing-label']
+
jobdesc['expires-after'] = test['expires-after']
jobdesc['routes'] = []
jobdesc['run-on-projects'] = test['run-on-projects']
jobdesc['scopes'] = []
jobdesc['tags'] = test.get('tags', {})
jobdesc['extra'] = {
'chunks': {
'current': test['this-chunk'],
--- a/taskcluster/taskgraph/util/attributes.py
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -14,16 +14,18 @@ TRUNK_PROJECTS = INTEGRATION_PROJECTS |
RELEASE_PROJECTS = {
'mozilla-central',
'mozilla-aurora',
'mozilla-beta',
'mozilla-release',
}
+_OPTIONAL_ATTRIBUTES = ('nightly', 'signed')
+
def attrmatch(attributes, **kwargs):
"""Determine whether the given set of task attributes matches. The
conditions are given as keyword arguments, where each keyword names an
attribute. The keyword value can be a literal, a set, or a callable. A
literal must match the attribute exactly. Given a set, the attribute value
must be in the set. A callable is called with the attribute value. If an
attribute is specified as a keyword argument but not present in the
@@ -76,8 +78,22 @@ def match_run_on_projects(project, run_o
if 'release' in run_on_projects:
if project in RELEASE_PROJECTS:
return True
if 'trunk' in run_on_projects:
if project in TRUNK_PROJECTS:
return True
return project in run_on_projects
+
+
+def copy_attributes_from_dependent_job(dep_job):
+ attributes = {
+ 'build_platform': dep_job.attributes.get('build_platform'),
+ 'build_type': dep_job.attributes.get('build_type'),
+ }
+
+ attributes.update({
+ attr: dep_job.attributes[attr]
+ for attr in _OPTIONAL_ATTRIBUTES if attr in dep_job.attributes
+ })
+
+ return attributes