--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -76,20 +76,20 @@ using :func:`taskgraph.transform.base.ge
Task-Generation Transforms
--------------------------
Every kind needs to create tasks, and all of those tasks have some things in
common. They all run on one of a small set of worker implementations, each
with their own idiosyncracies. And they all report to TreeHerder in a similar
way.
-The transforms in ``taskcluster/taskgraph/transforms/make_task.py`` implement
+The transforms in ``taskcluster/taskgraph/transforms/task.py`` implement
this common functionality. They expect a "task description", and produce a
task definition. The schema for a task description is defined at the top of
-``make_task.py``, with copious comments. The parts of the task description
+``task.py``, with copious comments. The parts of the task description
that are specific to a worker implementation are isolated in a ``worker``
object which has an ``implementation`` property naming the worker
implementation. Thus the transforms that produce a task description must be
aware of the worker implementation to be used, but need not be aware of the
details of its payload format.
The result is a dictionary with keys ``label``, ``attributes``, ``task``, and
``dependencies``, with the latter having the same format as the input
@@ -125,13 +125,13 @@ configuration in ``kind.yml``:
the "shape" of the test description, and are still governed by the schema in
``test_description.py``.
* The ``taskgraph.transforms.tests.make_task_description:transforms`` then
take the test description and create a *task* description. This transform
embodies the specifics of how test runs work: invoking mozharness, various
worker options, and so on.
- * Finally, the ``taskgraph.transforms.make_task:transforms``, described above
+ * Finally, the ``taskgraph.transforms.task:transforms``, described above
under "Task-Generation Transforms", are applied.
Test dependencies are produced in the form of a dictionary mapping dependency
name to task label.
rename from taskcluster/taskgraph/transforms/make_task.py
rename to taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/make_task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -3,63 +3,67 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
These transformations take a task description and turn it into a TaskCluster
task definition (along with attributes, label, etc.). The input to these
transformations is generic to any kind of task, but abstracts away some of the
complexities of worker implementations, scopes, and treeherder annotations.
"""
+from __future__ import absolute_import, print_function, unicode_literals
+
from taskgraph.util.treeherder import split_symbol
from taskgraph.transforms.base import (
validate_schema,
TransformSequence
)
from voluptuous import Schema, Any, Required, Optional, Extra
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
# A task description is a general description of a TaskCluster task
task_description_schema = Schema({
# the label for this task
- 'label': basestring,
+ Required('label'): basestring,
# description of the task (for metadata)
- 'description': basestring,
+ Required('description'): basestring,
# attributes for this task
- 'attributes': {basestring: object},
+ Optional('attributes'): {basestring: object},
# dependencies of this task, keyed by name; these are passed through
# verbatim and subject to the interpretation of the Task's get_dependencies
# method.
- 'dependencies': {basestring: object},
+ Optional('dependencies'): {basestring: object},
# expiration and deadline times, relative to task creation, with units
- # (e.g., "14 days")
- 'expires-after': basestring,
- 'deadline-after': basestring,
+ # (e.g., "14 days"). Defaults are set based on the project.
+ Optional('expires-after'): basestring,
+ Optional('deadline-after'): basestring,
# custom routes for this task; the default treeherder routes will be added
# automatically
- 'routes': [basestring],
+ Optional('routes'): [basestring],
# custom scopes for this task; any scopes required for the worker will be
# added automatically
- 'scopes': [basestring],
+ Optional('scopes'): [basestring],
# custom "task.extra" content
- 'extra': {basestring: object},
+ Optional('extra'): {basestring: object},
# treeherder-related information; see
# https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json
- 'treeherder': {
+ # If not specified, no treeherder extra information or routes will be
+ # added to the task
+ Optional('treeherder'): {
# either a bare symbol, or "grp(sym)".
'symbol': basestring,
# the job kind
'kind': Any('build', 'test', 'other'),
# tier for this task
'tier': int,
@@ -73,89 +77,92 @@ task_description_schema = Schema({
Required('environments', default=['production', 'staging']): ['production', 'staging'],
},
# the provisioner-id/worker-type for the task
'worker-type': basestring,
# information specific to the worker implementation that will run this task
'worker': Any({
- 'implementation': Any('docker-worker', 'docker-engine'),
+ Required('implementation'): Any('docker-worker', 'docker-engine'),
# the docker image (in docker's `host/repo/image:tag` format) in which
# to run the task; if omitted, this will be a reference to the image
# generated by the 'docker-image' dependency, which must be defined in
# 'dependencies'
Optional('docker-image'): basestring,
# worker features that should be enabled
Required('relengapi-proxy', default=False): bool,
+ Required('taskcluster-proxy', default=False): bool,
Required('allow-ptrace', default=False): bool,
Required('loopback-video', default=False): bool,
Required('loopback-audio', default=False): bool,
+ Optional('superseder-url'): basestring,
# caches to set up for the task
- 'caches': [{
+ Optional('caches'): [{
# only one type is supported by any of the workers right now
'type': 'persistent',
# name of the cache, allowing re-use by subsequent tasks naming the
# same cache
'name': basestring,
# location in the task image where the cache will be mounted
'mount-point': basestring,
}],
# artifacts to extract from the task image after completion
- 'artifacts': [{
+ Optional('artifacts'): [{
# type of artifact -- simple file, or recursive directory
'type': Any('file', 'directory'),
# task image path from which to read artifact
'path': basestring,
# name of the produced artifact (root of the names for
# type=directory)
'name': basestring,
}],
# environment variables
- 'env': {basestring: taskref_or_string},
+ Required('env', default={}): {basestring: taskref_or_string},
# the command to run
'command': [taskref_or_string],
# the maximum time to run, in seconds
'max-run-time': int,
}, {
- 'implementation': 'generic-worker',
+ Required('implementation'): 'generic-worker',
# command is a list of commands to run, sequentially
'command': [basestring],
# artifacts to extract from the task image after completion; note that artifacts
# for the generic worker cannot have names
- 'artifacts': [{
+ Optional('artifacts'): [{
# type of artifact -- simple file, or recursive directory
'type': Any('file', 'directory'),
# task image path from which to read artifact
'path': basestring,
}],
# environment variables
- 'env': {basestring: taskref_or_string},
+ Required('env', default={}): {basestring: taskref_or_string},
# the maximum time to run, in seconds
'max-run-time': int,
}, {
- 'implementation': 'buildbot-bridge',
+ Required('implementation'): 'buildbot-bridge',
- # see https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
+ # see
+ # https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
'buildername': basestring,
'sourcestamp': {
'branch': basestring,
Optional('revision'): basestring,
Optional('repository'): basestring,
Optional('project'): basestring,
},
'properties': {
@@ -293,47 +300,47 @@ def validate(config, tasks):
task_description_schema, task,
"In task {!r}:".format(task.get('label', '?no-label?')))
@transforms.add
def build_task(config, tasks):
for task in tasks:
provisioner_id, worker_type = task['worker-type'].split('/', 1)
- routes = task['routes']
- scopes = task['scopes']
+ routes = task.get('routes', [])
+ scopes = task.get('scopes', [])
# set up extra
- extra = task['extra']
- extra['treeherderEnv'] = task['treeherder']['environments']
-
- task_th = task['treeherder']
- treeherder = extra.setdefault('treeherder', {})
+ extra = task.get('extra', {})
+ task_th = task.get('treeherder')
+ if task_th:
+ extra['treeherderEnv'] = task_th['environments']
- machine_platform, collection = task_th['platform'].split('/', 1)
- treeherder['machine'] = {'platform': machine_platform}
- treeherder['collection'] = {collection: True}
+ treeherder = extra.setdefault('treeherder', {})
- groupSymbol, symbol = split_symbol(task_th['symbol'])
- if groupSymbol != '?':
+ machine_platform, collection = task_th['platform'].split('/', 1)
+ treeherder['machine'] = {'platform': machine_platform}
+ treeherder['collection'] = {collection: True}
+
+ groupSymbol, symbol = split_symbol(task_th['symbol'])
treeherder['groupSymbol'] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
- treeherder['symbol'] = symbol
- treeherder['jobKind'] = task_th['kind']
- treeherder['tier'] = task_th['tier']
+ treeherder['symbol'] = symbol
+ treeherder['jobKind'] = task_th['kind']
+ treeherder['tier'] = task_th['tier']
- routes.extend([
- '{}.v2.{}.{}.{}'.format(root,
- config.params['project'],
- config.params['head_rev'],
- config.params['pushlog_id'])
- for root in 'tc-treeherder', 'tc-treeherder-stage'
- ])
+ routes.extend([
+ '{}.v2.{}.{}.{}'.format(root,
+ config.params['project'],
+ config.params['head_rev'],
+ config.params['pushlog_id'])
+ for root in 'tc-treeherder', 'tc-treeherder-stage'
+ ])
task_def = {
'provisionerId': provisioner_id,
'workerType': worker_type,
'routes': routes,
'created': {'relative-datestamp': '0 seconds'},
'deadline': {'relative-datestamp': task['deadline-after']},
'expires': {'relative-datestamp': task['expires-after']},