Bug 1397847: allow jobs-from to be topsrcdir-relative, move toolchains; r?glandium
This moves the toolchain definitions out of the taskcluster/ci tree, in
preparation for making them useful outside of task-graph generation.
MozReview-Commit-ID: 928jAuvt2LM
rename from taskcluster/ci/toolchain/linux.yml
rename to build/toolchains/linux.yml
rename from taskcluster/ci/toolchain/macosx.yml
rename to build/toolchains/macosx.yml
rename from taskcluster/ci/toolchain/windows.yml
rename to build/toolchains/windows.yml
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -6,11 +6,11 @@ loader: taskgraph.loader.transform:loade
transforms:
- taskgraph.transforms.try_job:transforms
- taskgraph.transforms.toolchain:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- - linux.yml
- - macosx.yml
- - windows.yml
+ - /build/toolchains/linux.yml
+ - /build/toolchains/macosx.yml
+ - /build/toolchains/windows.yml
--- a/taskcluster/taskgraph/loader/transform.py
+++ b/taskcluster/taskgraph/loader/transform.py
@@ -1,31 +1,34 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
+import os
+from taskgraph import GECKO
from ..util.templates import merge
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
def loader(kind, path, config, params, loaded_tasks):
"""
Get the input elements that will be transformed into tasks in a generic
way. The elements themselves are free-form, and become the input to the
first transform.
- By default, this reads jobs from the `jobs` key, or from yaml files
- named by `jobs-from`. The entities are read from mappings, and the
- keys to those mappings are added in the `name` key of each entity.
+ By default, this reads jobs from the `jobs` key, or from yaml files named
+ by `jobs-from` (with absolute paths treated as relative to topsrcdir). The
+ entities are read from mappings, and the keys to those mappings are added
+ in the `name` key of each entity.
If there is a `job-defaults` config, then every job is merged with it.
This provides a simple way to set default values for all jobs of a
kind. More complex defaults should be implemented with custom
transforms.
Other kind implementations can use a different loader function to
produce inputs and hand them to `transform_inputs`.
@@ -34,17 +37,20 @@ def loader(kind, path, config, params, l
defaults = config.get('job-defaults')
for name, job in config.get('jobs', {}).iteritems():
if defaults:
job = merge(defaults, job)
job['job-from'] = 'kind.yml'
yield name, job
for filename in config.get('jobs-from', []):
- for name, job in load_yaml(path, filename).iteritems():
+ full_filename = filename
+ if filename[0] == '/':
+ full_filename = os.path.join(GECKO, filename[1:])
+ for name, job in load_yaml(path, full_filename).iteritems():
if defaults:
job = merge(defaults, job)
job['job-from'] = filename
yield name, job
for name, job in jobs():
job['name'] = name
logger.debug("Generating tasks for {} {}".format(kind, name))
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -69,17 +69,17 @@ def validate(config, jobs):
@transforms.add
def set_job_name(config, jobs):
for job in jobs:
job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
if 'job-from' in job and job['job-from'] != 'kind.yml':
- from_name = os.path.splitext(job['job-from'])[0]
+ from_name = os.path.splitext(os.path.basename(job['job-from']))[0]
job['name'] = '{}-{}'.format(from_name, job['name'])
yield job
@transforms.add
def expand_platforms(config, jobs):
for job in jobs:
if isinstance(job['platform'], basestring):