Bug 1318438 - [taskcluster] "job" tasks should have ability to run on multiple platforms, r?dustin
This adds an optional "platforms" key to the job description. It can be used in conjunction with
"by-platform" like so:
platforms:
- linux
- windows
worker-type:
by-platform:
linux: ...
windows: ...
worker:
by-platform:
linux: ...
windows: ...
MozReview-Commit-ID: JwL1NAR4bnY
--- a/taskcluster/docs/transforms.rst
+++ b/taskcluster/docs/transforms.rst
@@ -135,18 +135,20 @@ A job description says what to run in th
``run`` section and all of the fields from a task description. The run section
has a ``using`` property that defines how this task should be run; for example,
``mozharness`` to run a mozharness script, or ``mach`` to run a mach command.
The remainder of the run section is specific to the run-using implementation.
The effect of a job description is to say "run this thing on this worker". The
job description must contain enough information about the worker to identify
the workerType and the implementation (docker-worker, generic-worker, etc.).
-Any other task-description information is passed along verbatim, although it is
-augmented by the run-using implementation.
+Alternatively, job descriptions can specify the ``platforms`` field in
+conjunction with the ``by-platform`` key to specify multiple workerTypes and
+implementations. Any other task-description information is passed along
+verbatim, although it is augmented by the run-using implementation.
The run-using implementations are all located in
``taskcluster/taskgraph/transforms/job``, along with the schemas for their
implementations. Those well-commented source files are the canonical
documentation for what constitutes a job description, and should be considered
part of the documentation.
Task Descriptions
--- a/taskcluster/taskgraph/test/test_transforms_base.py
+++ b/taskcluster/taskgraph/test/test_transforms_base.py
@@ -103,41 +103,41 @@ class TestKeyedBy(unittest.TestCase):
'a': 10,
'default': 30,
},
},
'other-value': 'xxx',
}
self.assertEqual(get_keyed_by(test, 'option', 'x'), 30)
- def test_by_value_invalid_dict(self):
+ def test_by_value_dict(self):
test = {
'test-name': 'tname',
'option': {
'by-something-else': {},
'by-other-value': {},
},
}
- self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+ self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
def test_by_value_invalid_no_default(self):
test = {
'test-name': 'tname',
'option': {
'by-other-value': {
'a': 10,
},
},
'other-value': 'b',
}
self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
- def test_by_value_invalid_no_by(self):
+ def test_by_value_no_by(self):
test = {
'test-name': 'tname',
'option': {
'other-value': {},
},
}
- self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+ self.assertEqual(get_keyed_by(test, 'option', 'x'), test['option'])
if __name__ == '__main__':
main()
--- a/taskcluster/taskgraph/transforms/base.py
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -96,31 +96,28 @@ def get_keyed_by(item, field, item_name,
value = item[field]
if not isinstance(value, dict):
return value
if subfield:
value = item[field][subfield]
if not isinstance(value, dict):
return value
- assert len(value) == 1, "Invalid attribute {} in {}".format(field, item_name)
keyed_by = value.keys()[0]
+ if len(value) > 1 or not keyed_by.startswith('by-'):
+ return value
+
values = value[keyed_by]
- if keyed_by.startswith('by-'):
- keyed_by = keyed_by[3:] # extract just the keyed-by field name
- if item[keyed_by] in values:
- return values[item[keyed_by]]
- for k in values.keys():
- if re.match(k, item[keyed_by]):
- return values[k]
- if 'default' in values:
- return values['default']
- for k in item[keyed_by], 'default':
- if k in values:
- return values[k]
- else:
- raise Exception(
- "Neither {} {} nor 'default' found while determining item {} in {}".format(
- keyed_by, item[keyed_by], field, item_name))
+ keyed_by = keyed_by[3:] # strip 'by-' off the keyed-by field name
+ if item[keyed_by] in values:
+ return values[item[keyed_by]]
+ for k in values.keys():
+ if re.match(k, item[keyed_by]):
+ return values[k]
+ if 'default' in values:
+ return values['default']
+ for k in item[keyed_by], 'default':
+ if k in values:
+ return values[k]
else:
raise Exception(
- "Invalid attribute {} keyed-by value {} in {}".format(
- field, keyed_by, item_name))
+ "Neither {} {} nor 'default' found while determining item {} in {}".format(
+ keyed_by, item[keyed_by], field, item_name))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -10,23 +10,24 @@ run-using handlers in `taskcluster/taskg
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
import os
-from taskgraph.transforms.base import validate_schema, TransformSequence
+from taskgraph.transforms.base import get_keyed_by, validate_schema, TransformSequence
from taskgraph.transforms.task import task_description_schema
from voluptuous import (
+ Any,
+ Extra,
Optional,
Required,
Schema,
- Extra,
)
logger = logging.getLogger(__name__)
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@@ -47,54 +48,95 @@ job_description_schema = Schema({
Optional('expires-after'): task_description_schema['expires-after'],
Optional('routes'): task_description_schema['routes'],
Optional('scopes'): task_description_schema['scopes'],
Optional('extra'): task_description_schema['extra'],
Optional('treeherder'): task_description_schema['treeherder'],
Optional('index'): task_description_schema['index'],
Optional('run-on-projects'): task_description_schema['run-on-projects'],
Optional('coalesce-name'): task_description_schema['coalesce-name'],
- Optional('worker-type'): task_description_schema['worker-type'],
Optional('needs-sccache'): task_description_schema['needs-sccache'],
- Required('worker'): task_description_schema['worker'],
Optional('when'): task_description_schema['when'],
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
+ Optional('platforms'): [basestring],
+ Required('worker-type'): Any(
+ task_description_schema['worker-type'],
+ {'by-platform': {basestring: task_description_schema['worker-type']}},
+ ),
+ Required('worker'): Any(
+ task_description_schema['worker'],
+ {'by-platform': {basestring: task_description_schema['worker']}},
+ ),
})
transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
yield validate_schema(job_description_schema, job,
"In job {!r}:".format(job['name']))
@transforms.add
+def expand_platforms(config, jobs):
+ for job in jobs:
+ if 'platforms' not in job:
+ yield job
+ continue
+
+ for platform in job['platforms']:
+ pjob = copy.deepcopy(job)
+ pjob['platform'] = platform
+ del pjob['platforms']
+
+ platform, buildtype = platform.rsplit('/', 1)
+ pjob['name'] = '{}-{}-{}'.format(pjob['name'], platform, buildtype)
+ yield pjob
+
+
+@transforms.add
+def resolve_keyed_by(config, jobs):
+ fields = [
+ 'worker-type',
+ 'worker',
+ ]
+
+ for job in jobs:
+ for field in fields:
+ job[field] = get_keyed_by(item=job, field=field, item_name=job['name'])
+ yield job
+
+
+@transforms.add
def make_task_description(config, jobs):
"""Given a build description, create a task description"""
# import plugin modules first, before iterating over jobs
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
job['label'] = '{}-{}'.format(config.kind, job['name'])
if job['name']:
del job['name']
+ if 'platform' in job:
+ if 'treeherder' in job:
+ job['treeherder']['platform'] = job['platform']
+ del job['platform']
taskdesc = copy.deepcopy(job)
# fill in some empty defaults to make run implementations easier
taskdesc.setdefault('attributes', {})
taskdesc.setdefault('dependencies', {})
taskdesc.setdefault('routes', [])
taskdesc.setdefault('scopes', [])