Bug 1373326 - Reformat repackage tasks taskgraph to support new input style. r=dustin
(For Landing more OSX Nightly Support from date to central)
MozReview-Commit-ID: F8Dl1502D4L
--- a/taskcluster/ci/repackage-l10n/kind.yml
+++ b/taskcluster/ci/repackage-l10n/kind.yml
@@ -2,15 +2,16 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- taskgraph.transforms.repackage_l10n:transforms
- taskgraph.transforms.repackage:transforms
+ - taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- nightly-l10n-signing
only-for-build-platforms:
- macosx64-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -1,15 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- taskgraph.transforms.repackage:transforms
+ - taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build-signing
only-for-build-platforms:
- macosx64-nightly/opt
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -90,17 +90,17 @@ job_description_schema = Schema({
transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
yield validate_schema(job_description_schema, job,
- "In job {!r}:".format(job['name']))
+ "In job {!r}:".format(job.get('name', job.get('label'))))
@transforms.add
def rewrite_when_to_optimization(config, jobs):
for job in jobs:
when = job.pop('when', {})
files_changed = when.get('files-changed')
if not files_changed:
@@ -128,17 +128,17 @@ def make_task_description(config, jobs):
"""Given a build description, create a task description"""
# import plugin modules first, before iterating over jobs
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
job['label'] = '{}-{}'.format(config.kind, job['name'])
- if job['name']:
+ if job.get('name'):
del job['name']
impl, os = worker_type_implementation(job['worker-type'])
worker = job.setdefault('worker', {})
assert 'implementation' not in worker
worker['implementation'] = impl
if os:
worker['os'] = os
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -7,31 +7,38 @@ worker implementation they operate on, a
consistency.
"""
from __future__ import absolute_import, print_function, unicode_literals
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
-def docker_worker_add_workspace_cache(config, job, taskdesc):
+def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
"""Add the workspace cache based on the build platform/type and level,
- except on try where workspace caches are not used."""
+ except on try where workspace caches are not used.
+
+ extra, is an optional kwarg passed in that supports extending the cache
+ key name to avoid undesired conflicts with other caches."""
if config.params['project'] == 'try':
return
taskdesc['worker'].setdefault('caches', []).append({
'type': 'persistent',
'name': 'level-{}-{}-build-{}-{}-workspace'.format(
config.params['level'], config.params['project'],
taskdesc['attributes']['build_platform'],
taskdesc['attributes']['build_type'],
),
'mount-point': "/home/worker/workspace",
})
+ if extra:
+ taskdesc['worker']['caches'][-1]['name'] += '-{}'.format(
+ extra
+ )
def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
taskdesc['worker'].setdefault('caches', []).append({
'type': 'persistent',
'name': 'level-{}-{}-tc-vcs'.format(
config.params['level'], config.params['project']),
'mount-point': "/home/worker/.tc-vcs",
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -44,16 +44,20 @@ mozharness_run_schema = Schema({
# any additional options (without leading --) to be passed to mozharness;
# not supported on Windows
Optional('options'): [basestring],
# --custom-build-variant-cfg value (not supported on Windows)
Optional('custom-build-variant-cfg'): basestring,
+ # Extra metadata to use toward the workspace caching.
+ # Only supported on docker-worker
+ Optional('extra-workspace-cache-key'): basestring,
+
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
'public',
'internal',
),
@@ -89,17 +93,18 @@ def mozharness_on_docker_worker_setup(co
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
- docker_worker_add_workspace_cache(config, job, taskdesc)
+ docker_worker_add_workspace_cache(config, job, taskdesc,
+ extra=run.get('extra-workspace-cache-key'))
support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -62,111 +62,104 @@ def validate(config, jobs):
@transforms.add
def make_repackage_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
label = job.get('label',
dep_job.label.replace("signing-", "repackage-"))
job['label'] = label
- cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
- cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
yield job
@transforms.add
-def make_task_description(config, jobs):
+def make_job_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
+ dependencies = {dep_job.attributes.get('kind'): dep_job.label}
+ if len(dep_job.dependencies) > 1:
+ raise NotImplementedError(
+ "Can't repackage a signing task with multiple dependencies")
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(Nr)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
+ signing_task = None
+ for dependency in dependencies.keys():
+ if 'signing' in dependency:
+ signing_task = dependency
+ signing_task_ref = "<{}>".format(signing_task)
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
+ if job.get('locale'):
+ attributes['locale'] = job['locale']
- command = ['/home/worker/bin/run-task',
- # Various caches/volumes are default owned by root:root.
- '--chown-recursive', '/home/worker/workspace',
- '--chown-recursive', '/home/worker/tooltool-cache',
- '--vcs-checkout', '/home/worker/workspace/build/src',
- '--tools-checkout', '/home/worker/workspace/build/tools',
- '--',
- '/home/worker/workspace/build/src/taskcluster/scripts/builder/repackage.sh'
- ]
+ level = config.params['level']
- dependencies = {dep_job.attributes.get('kind'): dep_job.label}
- if job.get('locale'):
- input_string = 'https://queue.taskcluster.net/v1/task/<nightly-l10n-signing>/' + \
- 'artifacts/public/build/{}/target.tar.gz'
- input_string = input_string.format(job['locale'])
+ task_env = {}
+ locale_output_path = ""
+ if attributes['build_platform'].startswith('macosx'):
+ if job.get('locale'):
+ input_string = 'https://queue.taskcluster.net/v1/task/' + \
+ '{}/artifacts/public/build/{}/target.tar.gz'
+ input_string = input_string.format(signing_task_ref, job['locale'])
+ locale_output_path = "{}/".format(job['locale'])
+ else:
+ input_string = 'https://queue.taskcluster.net/v1/task/' + \
+ '{}/artifacts/public/build/target.tar.gz'.format(signing_task_ref)
+ task_env.update(
+ SIGNED_INPUT={'task-reference': input_string},
+ )
+ mozharness_config = ['repackage/osx_signed.py']
+ output_files = [{
+ 'type': 'file',
+ 'path': '/home/worker/workspace/build/artifacts/target.dmg',
+ 'name': 'public/build/{}target.dmg'.format(locale_output_path),
+ }]
else:
- input_string = 'https://queue.taskcluster.net/v1/task/<build-signing>/artifacts/' + \
- 'public/build/target.tar.gz'
- signed_input = {'task-reference': input_string}
- level = config.params['level']
+ raise Exception("Unexpected build platform for repackage")
+
+ run = {
+ 'using': 'mozharness',
+ 'script': 'mozharness/scripts/repackage.py',
+ 'config': mozharness_config,
+ 'job-script': 'taskcluster/scripts/builder/repackage.sh',
+ 'actions': ['download_input', 'setup', 'repackage'],
+ 'extra-workspace-cache-key': 'repackage',
+ }
+
+ if attributes["build_platform"].startswith('macosx'):
+ worker = {
+ 'docker-image': {"in-tree": "desktop-build"},
+ 'artifacts': output_files,
+ 'env': task_env,
+ 'chain-of-trust': True,
+ 'max-run-time': 3600
+ }
+ run["tooltool-downloads"] = 'internal'
+ worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
+ cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+ cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
task = {
'label': job['label'],
'description': "{} Repackage".format(
dep_job.task["metadata"]["description"]),
- 'worker-type': 'aws-provisioner-v1/gecko-%s-b-macosx64' % level,
+ 'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
- 'scopes':
- ['docker-worker:relengapi-proxy:tooltool.download.internal',
- 'secrets:get:project/taskcluster/gecko/hgfingerprint',
- 'docker-worker:relengapi-proxy:tooltool.download.public',
- 'project:releng:signing:format:dmg'],
- 'worker': {'implementation': 'docker-worker',
- 'os': 'linux',
- 'docker-image': {"in-tree": "desktop-build"},
- 'caches': [{
- 'type': 'persistent',
- 'name': 'tooltool-cache',
- 'mount-point': '/home/worker/tooltool-cache',
- }, {
- 'type': 'persistent',
- 'name': 'level-%s-checkouts-v1' % level,
- 'mount-point': '/home/worker/checkouts',
- }],
- 'artifacts': [{
- 'type': 'file',
- 'path': '/home/worker/workspace/build/upload/target.dmg',
- 'name': 'public/build/target.dmg',
- }],
- 'env': {
- 'SIGNED_INPUT': signed_input,
- 'MOZ_BUILD_DATE': config.params['moz_build_date'],
- 'MH_BUILD_POOL': 'taskcluster',
- 'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
- 'GECKO_HEAD_REV': config.params['head_rev'],
- 'MH_BRANCH': config.params['project'],
- 'MOZ_SCM_LEVEL': level,
- 'MOZHARNESS_ACTIONS': 'download_input setup repackage',
- 'NEED_XVFB': 'true',
- 'GECKO_BASE_REPOSITORY': config.params['base_repository'],
- 'TOOLTOOL_CACHE': '/home/worker/tooltool-cache',
- 'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
- 'MOZHARNESS_CONFIG': 'repackage/osx_signed.py',
- 'USE_SCCACHE': '1',
- 'MOZHARNESS_SCRIPT': 'mozharness/scripts/repackage.py'
- },
- 'command': command,
- 'chain-of-trust': True,
- 'relengapi-proxy': True,
- 'max-run-time': 3600
- }
+ 'worker': worker,
+ 'run': run,
}
yield task
--- a/taskcluster/taskgraph/transforms/repackage_l10n.py
+++ b/taskcluster/taskgraph/transforms/repackage_l10n.py
@@ -12,20 +12,22 @@ from taskgraph.transforms.base import Tr
transforms = TransformSequence()
@transforms.add
def split_locales(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
for locale in dep_job.attributes.get('chunk_locales', []):
- job['locale'] = locale
-
label = dep_job.label.replace("signing-", "repackage-{}-".format(locale))
label = "repackage-{}-{}/{}".format(locale,
dep_job.attributes['build_platform'],
dep_job.attributes['build_type'],)
- job['label'] = label
treeherder = job.get('treeherder', {})
treeherder['symbol'] = 'tc-L10n-Rpk({})'.format(locale)
- job['treeherder'] = treeherder
- yield job
+
+ yield {
+ 'locale': locale,
+ 'label': label,
+ 'treeherder': treeherder,
+ 'dependent-task': dep_job,
+ }