Bug 1336559 - Add ability to depend on build artifacts to 'run_task' based tasks, r?dustin
Currently 'run_task' tasks have no easy way to depend on a build task. For example, some
python unittests need a Firefox binary for their tests, like the mozrunner tests and future
test harness selftests (like mochitest tests).
This patch allows kinds to add a new key to the kind config which maps test platforms to
build-labels. Then 'run_task' jobs can add a 'requires-build': true field to get a build
dependency automatically added. The build artifact url will also be stored in the
$GECKO_INSTALLER_URL environment variable on the test host.
MozReview-Commit-ID: Jqyhbj7nC6z
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -9,8 +9,13 @@ transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- python-tests.yml
- mozlint.yml
- doc.yml
- webidl.yml
+
+# This is used by run-task based tasks to lookup which build task it
+# should depend on based on its own platform.
+dependent-build-platforms:
+ linux64.*: build-linux64/opt
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -127,35 +127,35 @@ def make_task_description(config, jobs):
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
job['label'] = '{}-{}'.format(config.kind, job['name'])
if job['name']:
del job['name']
- if 'platform' in job:
- if 'treeherder' in job:
- job['treeherder']['platform'] = job['platform']
- del job['platform']
-
taskdesc = copy.deepcopy(job)
# fill in some empty defaults to make run implementations easier
taskdesc.setdefault('attributes', {})
taskdesc.setdefault('dependencies', {})
taskdesc.setdefault('routes', [])
taskdesc.setdefault('scopes', [])
taskdesc.setdefault('extra', {})
# give the function for job.run.using on this worker implementation a
# chance to set up the task description.
configure_taskdesc_for_run(config, job, taskdesc)
del taskdesc['run']
+ if 'platform' in taskdesc:
+ if 'treeherder' in taskdesc:
+ taskdesc['treeherder']['platform'] = taskdesc['platform']
+ del taskdesc['platform']
+
# yield only the task description, discarding the job description
yield taskdesc
# A registry of all functions decorated with run_job_using
registry = {}
def run_job_using(worker_implementation, run_using, schema=None):
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -4,16 +4,20 @@
"""
Common support for various job types. These functions are all named after the
worker implementation they operate on, and take the same three parameters, for
consistency.
"""
from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.util.attributes import keymatch
+
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
def docker_worker_add_workspace_cache(config, job, taskdesc):
"""Add the workspace cache based on the build platform/type and level,
except on try where workspace caches are not used."""
if config.params['project'] == 'try':
return
@@ -52,16 +56,45 @@ def docker_worker_add_gecko_vcs_env_vars
env.update({
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
'GECKO_HEAD_REF': config.params['head_rev'],
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
})
+def docker_worker_add_build_dependency(config, job, taskdesc):
+ """Add build dependency to the task description and installer_url to env."""
+ key = job['platform']
+ build_labels = config.config.get('dependent-build-platforms', {})
+ matches = keymatch(build_labels, key)
+ if not matches:
+ raise Exception("No build platform found for '{}'. "
+ "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+ if len(matches) > 1:
+ raise Exception("More than one build platform found for '{}'.".format(key))
+
+ label = matches[0]
+ deps = taskdesc.setdefault('dependencies', {})
+ deps.update({'build': label})
+
+ if 'macosx' in label:
+ target = 'target.dmg'
+ elif 'android' in label:
+ target = 'target.apk'
+ else:
+ target = 'target.tar.bz2'
+ build_artifact = 'public/build/{}'.format(target)
+ installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+ env = taskdesc['worker'].setdefault('env', {})
+ env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
def docker_worker_support_vcs_checkout(config, job, taskdesc):
"""Update a job/task with parameters to enable a VCS checkout.
The configuration is intended for tasks using "run-task" and its
VCS checkout behavior.
"""
level = config.params['level']
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -11,16 +11,22 @@ from taskgraph.transforms.job import run
from taskgraph.transforms.job.run_task import docker_worker_run_task
from voluptuous import Schema, Required
mach_schema = Schema({
Required('using'): 'mach',
# The mach command (omitting `./mach`) to run
Required('mach'): basestring,
+
+ # Whether the job requires a build artifact or not. If True, the task
+ # will depend on a build task and run-task will download and set up the
+ # installer. Build labels are determined by the `dependent-build-platforms`
+ # config in kind.yml.
+ Required('requires-build', default=False): bool,
})
@run_job_using("docker-worker", "mach", schema=mach_schema)
def docker_worker_mach(config, job, taskdesc):
run = job['run']
# defer to the run_task implementation
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -6,16 +6,17 @@ Support for running jobs that are invoke
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
+ docker_worker_add_build_dependency,
docker_worker_support_vcs_checkout,
)
from voluptuous import Schema, Required, Any
run_task_schema = Schema({
Required('using'): 'run-task',
# if true, add a cache at ~worker/.cache, which is where things like pip
@@ -24,28 +25,37 @@ run_task_schema = Schema({
# if true (the default), perform a checkout in /home/worker/checkouts/gecko
Required('checkout', default=True): bool,
# The command arguments to pass to the `run-task` script, after the
# checkout arguments. If a list, it will be passed directly; otherwise
# it will be included in a single argument to `bash -cx`.
Required('command'): Any([basestring], basestring),
+
+ # Whether the job requires a build artifact or not. If True, the task
+ # will depend on a build task and run-task will download and set up the
+ # installer. Build labels are determined by the `dependent-build-platforms`
+ # config in kind.yml.
+ Required('requires-build', default=False): bool,
})
@run_job_using("docker-worker", "run-task", schema=run_task_schema)
def docker_worker_run_task(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker'] = copy.deepcopy(job['worker'])
if run['checkout']:
docker_worker_support_vcs_checkout(config, job, taskdesc)
+ if run['requires-build']:
+ docker_worker_add_build_dependency(config, job, taskdesc)
+
if run.get('cache-dotcache') and int(config.params['level']) > 1:
worker['caches'].append({
'type': 'persistent',
'name': 'level-{level}-{project}-dotcache'.format(**config.params),
'mount-point': '/home/worker/.cache',
})
run_command = run['command']