Bug 1397847: allow jobs-from to be topsrcdir-relative, move toolchains; r?glandium draft
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 14 Sep 2017 20:11:57 +0000
changeset 665125 84547dc57180fe679ed35f5cd111816db1eaae63
parent 664064 e5f80a639bfe68b68693a5be610f9d36b6c5ad00
child 665126 4d29e16b186e836db589c6b80cb7b50631e8ebc4
push id79939
push userdmitchell@mozilla.com
push dateThu, 14 Sep 2017 23:47:12 +0000
reviewersglandium
bugs1397847
milestone57.0a1
Bug 1397847: allow jobs-from to be topsrcdir-relative, move toolchains; r?glandium This moves the toolchain definitions out of the taskcluster/ci tree, in preparation for making them useful outside of task-graph generation. MozReview-Commit-ID: 928jAuvt2LM
build/toolchains/linux.yml
build/toolchains/macosx.yml
build/toolchains/windows.yml
taskcluster/ci/toolchain/kind.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/taskgraph/loader/transform.py
taskcluster/taskgraph/transforms/source_test.py
rename from taskcluster/ci/toolchain/linux.yml
rename to build/toolchains/linux.yml
rename from taskcluster/ci/toolchain/macosx.yml
rename to build/toolchains/macosx.yml
rename from taskcluster/ci/toolchain/windows.yml
rename to build/toolchains/windows.yml
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -6,11 +6,11 @@ loader: taskgraph.loader.transform:loade
 
 transforms:
    - taskgraph.transforms.try_job:transforms
    - taskgraph.transforms.toolchain:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
-   - linux.yml
-   - macosx.yml
-   - windows.yml
+   - /build/toolchains/linux.yml
+   - /build/toolchains/macosx.yml
+   - /build/toolchains/windows.yml
--- a/taskcluster/taskgraph/loader/transform.py
+++ b/taskcluster/taskgraph/loader/transform.py
@@ -1,31 +1,34 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
+import os
 
+from taskgraph import GECKO
 from ..util.templates import merge
 from ..util.yaml import load_yaml
 
 logger = logging.getLogger(__name__)
 
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Get the input elements that will be transformed into tasks in a generic
     way.  The elements themselves are free-form, and become the input to the
     first transform.
 
-    By default, this reads jobs from the `jobs` key, or from yaml files
-    named by `jobs-from`.  The entities are read from mappings, and the
-    keys to those mappings are added in the `name` key of each entity.
+    By default, this reads jobs from the `jobs` key, or from yaml files named
+    by `jobs-from` (with absolute paths treated as relative to topsrcdir).  The
+    entities are read from mappings, and the keys to those mappings are added
+    in the `name` key of each entity.
 
     If there is a `job-defaults` config, then every job is merged with it.
     This provides a simple way to set default values for all jobs of a
     kind.  More complex defaults should be implemented with custom
     transforms.
 
     Other kind implementations can use a different loader function to
     produce inputs and hand them to `transform_inputs`.
@@ -34,17 +37,20 @@ def loader(kind, path, config, params, l
         defaults = config.get('job-defaults')
         for name, job in config.get('jobs', {}).iteritems():
             if defaults:
                 job = merge(defaults, job)
             job['job-from'] = 'kind.yml'
             yield name, job
 
         for filename in config.get('jobs-from', []):
-            for name, job in load_yaml(path, filename).iteritems():
+            full_filename = filename
+            if filename[0] == '/':
+                full_filename = os.path.join(GECKO, filename[1:])
+            for name, job in load_yaml(path, full_filename).iteritems():
                 if defaults:
                     job = merge(defaults, job)
                 job['job-from'] = filename
                 yield name, job
 
     for name, job in jobs():
         job['name'] = name
         logger.debug("Generating tasks for {} {}".format(kind, name))
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -69,17 +69,17 @@ def validate(config, jobs):
 
 
 @transforms.add
 def set_job_name(config, jobs):
     for job in jobs:
         job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
 
         if 'job-from' in job and job['job-from'] != 'kind.yml':
-            from_name = os.path.splitext(job['job-from'])[0]
+            from_name = os.path.splitext(os.path.basename(job['job-from']))[0]
             job['name'] = '{}-{}'.format(from_name, job['name'])
         yield job
 
 
 @transforms.add
 def expand_platforms(config, jobs):
     for job in jobs:
         if isinstance(job['platform'], basestring):