Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r?dustin
The 'platform' key was recently moved out of 'job' tasks and into the 'source-test' kind. Since
the concept of requiring a build depends on this key, let's move that back to source-test as
well.
MozReview-Commit-ID: 4bs8G4wN5OH
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -4,20 +4,16 @@
"""
Common support for various job types. These functions are all named after the
worker implementation they operate on, and take the same three parameters, for
consistency.
"""
from __future__ import absolute_import, print_function, unicode_literals
-from taskgraph.util.attributes import keymatch
-
-
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
def docker_worker_add_workspace_cache(config, job, taskdesc):
"""Add the workspace cache based on the build platform/type and level,
except on try where workspace caches are not used."""
if config.params['project'] == 'try':
return
@@ -56,40 +52,16 @@ def docker_worker_add_gecko_vcs_env_vars
env.update({
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
'GECKO_HEAD_REF': config.params['head_rev'],
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
})
-def add_build_dependency(config, job, taskdesc):
- """Add build dependency to the task description and installer_url to env."""
- key = job['platform']
- build_labels = config.config.get('dependent-build-platforms', {})
- matches = keymatch(build_labels, key)
- if not matches:
- raise Exception("No build platform found for '{}'. "
- "Define 'dependent-build-platforms' in the kind config.".format(key))
-
- if len(matches) > 1:
- raise Exception("More than one build platform found for '{}'.".format(key))
-
- label = matches[0]['label']
- target = matches[0]['target-name']
- deps = taskdesc.setdefault('dependencies', {})
- deps.update({'build': label})
-
- build_artifact = 'public/build/{}'.format(target)
- installer_url = ARTIFACT_URL.format('<build>', build_artifact)
-
- env = taskdesc['worker'].setdefault('env', {})
- env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
-
-
def support_vcs_checkout(config, job, taskdesc):
"""Update a job/task with parameters to enable a VCS checkout.
The configuration is intended for tasks using "run-task" and its
VCS checkout behavior.
"""
level = config.params['level']
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -15,22 +15,16 @@ from taskgraph.transforms.job.run_task i
from taskgraph.util.schema import Schema
from voluptuous import Required
mach_schema = Schema({
Required('using'): 'mach',
# The mach command (omitting `./mach`) to run
Required('mach'): basestring,
-
- # Whether the job requires a build artifact or not. If True, the task
- # will depend on a build task and run-task will download and set up the
- # installer. Build labels are determined by the `dependent-build-platforms`
- # config in kind.yml.
- Required('requires-build', default=False): bool,
})
@run_job_using("docker-worker", "mach", schema=mach_schema)
@run_job_using("native-engine", "mach", schema=mach_schema)
def docker_worker_mach(config, job, taskdesc):
run = job['run']
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -4,53 +4,41 @@
"""
Support for running jobs that are invoked via the `run-task` script.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using
from taskgraph.util.schema import Schema
-from taskgraph.transforms.job.common import (
- add_build_dependency,
- support_vcs_checkout,
-)
+from taskgraph.transforms.job.common import support_vcs_checkout
from voluptuous import Required, Any
run_task_schema = Schema({
Required('using'): 'run-task',
# if true, add a cache at ~worker/.cache, which is where things like pip
# tend to hide their caches. This cache is never added for level-1 jobs.
Required('cache-dotcache', default=False): bool,
# if true (the default), perform a checkout in /home/worker/checkouts/gecko
Required('checkout', default=True): bool,
# The command arguments to pass to the `run-task` script, after the
# checkout arguments. If a list, it will be passed directly; otherwise
# it will be included in a single argument to `bash -cx`.
Required('command'): Any([basestring], basestring),
-
- # Whether the job requires a build artifact or not. If True, the task
- # will depend on a build task and run-task will download and set up the
- # installer. Build labels are determined by the `dependent-build-platforms`
- # config in kind.yml.
- Required('requires-build', default=False): bool,
})
def common_setup(config, job, taskdesc):
run = job['run']
if run['checkout']:
support_vcs_checkout(config, job, taskdesc)
- if run['requires-build']:
- add_build_dependency(config, job, taskdesc)
-
@run_job_using("docker-worker", "run-task", schema=run_task_schema)
def docker_worker_run_task(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker'] = job['worker']
common_setup(config, job, taskdesc)
if run.get('cache-dotcache') and int(config.params['level']) > 1:
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -8,39 +8,48 @@ treeherder configuration and attributes
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.job import job_description_schema
+from taskgraph.util.attributes import keymatch
from taskgraph.util.schema import (
validate_schema,
resolve_keyed_by,
)
from voluptuous import (
Any,
Extra,
Required,
Schema,
)
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
source_test_description_schema = Schema({
# most fields are passed directly through as job fields, and are not
# repeated here
Extra: object,
# The platform on which this task runs. This will be used to set up attributes
# (for try selection) and treeherder metadata (for display). If given as a list,
# the job will be "split" into multiple tasks, one with each platform.
Required('platform'): Any(basestring, [basestring]),
+ # Whether the job requires a build artifact or not. If True, the task will
+ # depend on a build task and the installer url will be saved to the
+ # GECKO_INSTALLER_URL environment variable. Build labels are determined by the
+ # `dependent-build-platforms` config in kind.yml.
+ Required('require-build', default=False): bool,
+
# These fields can be keyed by "platform", and are otherwise identical to
# job descriptions.
Required('worker-type'): Any(
job_description_schema['worker-type'],
{'by-platform': {basestring: job_description_schema['worker-type']}},
),
Required('worker'): Any(
job_description_schema['worker'],
@@ -78,16 +87,42 @@ def expand_platforms(config, jobs):
if 'name' in pjob:
pjob['name'] = '{}-{}'.format(pjob['name'], platform)
else:
pjob['label'] = '{}-{}'.format(pjob['label'], platform)
yield pjob
+def add_build_dependency(config, job):
+ """
+ Add build dependency to the job and installer_url to env.
+ """
+ key = job['platform']
+ build_labels = config.config.get('dependent-build-platforms', {})
+ matches = keymatch(build_labels, key)
+ if not matches:
+ raise Exception("No build platform found for '{}'. "
+ "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+ if len(matches) > 1:
+ raise Exception("More than one build platform found for '{}'.".format(key))
+
+ label = matches[0]['label']
+ target = matches[0]['target-name']
+ deps = job.setdefault('dependencies', {})
+ deps.update({'build': label})
+
+ build_artifact = 'public/build/{}'.format(target)
+ installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+ env = job['worker'].setdefault('env', {})
+ env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
@transforms.add
def handle_platform(config, jobs):
"""
Handle the 'platform' property, setting up treeherder context as well as
try-related attributes.
"""
fields = [
'worker-type',
@@ -98,10 +133,13 @@ def handle_platform(config, jobs):
platform = job['platform']
for field in fields:
resolve_keyed_by(job, field, item_name=job['name'])
if 'treeherder' in job:
job['treeherder']['platform'] = platform
+ if job.pop('require-build'):
+ add_build_dependency(config, job)
+
del job['platform']
yield job