Bug 1286075: allow defaults in TransformTask; r=gps
MozReview-Commit-ID: 5tdnxykLTZc
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -1,19 +1,21 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
+import itertools
from . import base
from .. import files_changed
from ..util.python_path import find_object
+from ..util.templates import merge
from ..util.yaml import load_yaml
from ..transforms.base import TransformSequence, TransformConfig
logger = logging.getLogger(__name__)
class TransformTask(base.Task):
"""
@@ -26,27 +28,37 @@ class TransformTask(base.Task):
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
"""
Get the input elements that will be transformed into tasks. The
elements themselves are free-form, and become the input to the first
transform.
By default, this reads jobs from the `jobs` key, or from yaml files
- named by `jobs-from`, but can be overridden in subclasses. The
- entities are read from mappings, and the keys to those mappings are
- added in the `name` key of each entity.
+ named by `jobs-from`. The entities are read from mappings, and the
+ keys to those mappings are added in the `name` key of each entity.
+
+ If there is a `job-defaults` config, then every job is merged with it.
+ This provides a simple way to set default values for all jobs of a
+ kind. More complex defaults should be implemented with custom
+ transforms.
+
+ This method can be overridden in subclasses that need to perform more
+ complex calculations to generate the list of inputs.
"""
def jobs():
- for name, job in config.get('jobs', {}).iteritems():
+ defaults = config.get('job-defaults')
+ jobs = config.get('jobs', {}).iteritems()
+ jobs_from = itertools.chain.from_iterable(
+ load_yaml(path, filename).iteritems()
+ for filename in config.get('jobs-from', {}))
+ for name, job in itertools.chain(jobs, jobs_from):
+ if defaults:
+ job = merge(defaults, job)
yield name, job
- for filename in config.get('jobs-from', {}):
- jobs = load_yaml(path, filename)
- for name, job in jobs.iteritems():
- yield name, job
for name, job in jobs():
job['name'] = name
logger.debug("Generating tasks for {} {}".format(kind, name))
yield job
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):