Bug 1395717 - Use filename source-test tasks are defined in as part of their label, r?dustin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 31 Aug 2017 16:38:08 -0400
changeset 659256 b51461fab56a03e94a3701dd559fb315b37e350e
parent 659065 3ecda4678c49ca255c38b1697142b9118cdd27e7
child 729938 7ca68f5b87f69e2b0b537fdb55708c03e2865869
push id78074
push userahalberstadt@mozilla.com
push dateTue, 05 Sep 2017 19:25:48 +0000
reviewersdustin
bugs1395717
milestone57.0a1
Bug 1395717 - Use filename source-test tasks are defined in as part of their label, r?dustin This creates a new "job-from" field that contains the relative filename the job was defined in. The filename is relative to 'config.path'. If the task came from the 'jobs' key defined in kind.yml, this field will be set to 'kind.yml'. MozReview-Commit-ID: 9e1tEb6XuZT
taskcluster/ci/source-test/doc.yml
taskcluster/ci/source-test/kind.yml
taskcluster/ci/source-test/mozlint.yml
taskcluster/ci/source-test/python-tests.yml
taskcluster/ci/source-test/python.yml
taskcluster/ci/source-test/webidl.yml
taskcluster/taskgraph/loader/transform.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/push_apk.py
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
taskcluster/taskgraph/transforms/source_test.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/source-test/doc.yml
+++ b/taskcluster/ci/source-test/doc.yml
@@ -1,9 +1,9 @@
-doc-generate:
+generate:
     description: Generate the Sphinx documentation
     platform: lint/opt
     treeherder:
         symbol: tc(Doc)
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
@@ -19,17 +19,17 @@ doc-generate:
             cd /builds/worker/checkouts/gecko &&
             ./mach doc --outdir docs-out --no-open --archive
     when:
         files-changed:
             - '**/*.py'
             - '**/*.rst'
             - 'tools/docs/**'
 
-doc-upload:
+upload:
     description: Generate and upload the Sphinx documentation
     platform: lint/opt
     treeherder:
         symbol: tc(DocUp)
         kind: test
         tier: 3
     run-on-projects: [mozilla-central]
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -9,17 +9,17 @@ transforms:
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
    - cram.yml
    - doc.yml
    - mocha.yml
    - mozlint.yml
-   - python-tests.yml
+   - python.yml
    - webidl.yml
 
 # This is used by run-task based tasks to lookup which build task it
 # should depend on based on its own platform.
 dependent-build-platforms:
    linux64-asan/opt:
       label: build-linux64-asan/opt
       target-name: target.tar.bz2
--- a/taskcluster/ci/source-test/mozlint.yml
+++ b/taskcluster/ci/source-test/mozlint.yml
@@ -1,9 +1,9 @@
-mozlint-eslint:
+eslint:
     description: JS lint check
     platform: lint/opt
     treeherder:
         symbol: ES
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
@@ -31,17 +31,37 @@ mozlint-eslint:
             - '**/*eslintrc*'
             # The plugin implementing custom checks.
             - 'tools/lint/eslint/eslint-plugin-mozilla/**'
             - 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'
             # Other misc lint related files.
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-mozlint-py-flake8:
+py-compat:
+    description: lint for python 2/3 compatibility issues
+    platform: lint/opt
+    treeherder:
+        symbol: py-compat
+        kind: test
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l py2 -l py3 -f treeherder
+    when:
+        files-changed:
+            - '**/*.py'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+
+py-flake8:
     description: flake8 run over the gecko codebase
     platform: lint/opt
     treeherder:
         symbol: f8
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
@@ -52,58 +72,16 @@ mozlint-py-flake8:
         mach: lint -l flake8 -f treeherder
     when:
         files-changed:
             - '**/*.py'
             - '**/.flake8'
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-mozlint-yaml:
-    description: yamllint run over the gecko codebase
-    platform: lint/opt
-    treeherder:
-        symbol: yaml
-        kind: test
-        tier: 1
-    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-    worker:
-        docker-image: {in-tree: "lint"}
-        max-run-time: 1800
-    run:
-        using: mach
-        mach: lint -l yaml -f treeherder
-    when:
-        files-changed:
-            - '**/*.yml'
-            - '**/*.yaml'
-            - '**/.ymllint'
-            - 'python/mozlint/**'
-            - 'tools/lint/**'
-
-mozlint-py-compat:
-    description: lint for python 2/3 compatibility issues
-    platform: lint/opt
-    treeherder:
-        symbol: py-compat
-        kind: test
-        tier: 1
-    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-    worker:
-        docker-image: {in-tree: "lint"}
-        max-run-time: 1800
-    run:
-        using: mach
-        mach: lint -l py2 -l py3 -f treeherder
-    when:
-        files-changed:
-            - '**/*.py'
-            - 'python/mozlint/**'
-            - 'tools/lint/**'
-
 wptlint-gecko:
     description: web-platform-tests linter
     platform: lint/opt
     treeherder:
         symbol: W
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
@@ -116,8 +94,30 @@ wptlint-gecko:
     when:
         files-changed:
             - 'testing/web-platform/tests/**'
             - 'testing/web-platform/mozilla/tests/**'
             - 'testing/web-platform/meta/MANIFEST.json'
             - 'testing/web-platform/mozilla/meta/MANIFEST.json'
             - 'python/mozlint/**'
             - 'tools/lint/**'
+
+yaml:
+    description: yamllint run over the gecko codebase
+    platform: lint/opt
+    treeherder:
+        symbol: yaml
+        kind: test
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l yaml -f treeherder
+    when:
+        files-changed:
+            - '**/*.yml'
+            - '**/*.yaml'
+            - '**/.ymllint'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
rename from taskcluster/ci/source-test/python-tests.yml
rename to taskcluster/ci/source-test/python.yml
--- a/taskcluster/ci/source-test/webidl.yml
+++ b/taskcluster/ci/source-test/webidl.yml
@@ -1,9 +1,9 @@
-webidl-test:
+test:
     description: WebIDL parser tests
     platform: lint/opt
     treeherder:
         symbol: Wp
         kind: test
         tier: 1
     worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
     worker:
--- a/taskcluster/taskgraph/loader/transform.py
+++ b/taskcluster/taskgraph/loader/transform.py
@@ -1,16 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
-import itertools
 
 from ..util.templates import merge
 from ..util.yaml import load_yaml
 
 logger = logging.getLogger(__name__)
 
 
 def loader(kind, path, config, params, loaded_tasks):
@@ -28,21 +27,25 @@ def loader(kind, path, config, params, l
     kind.  More complex defaults should be implemented with custom
     transforms.
 
     Other kind implementations can use a different loader function to
     produce inputs and hand them to `transform_inputs`.
     """
     def jobs():
         defaults = config.get('job-defaults')
-        jobs = config.get('jobs', {}).iteritems()
-        jobs_from = itertools.chain.from_iterable(
-            load_yaml(path, filename).iteritems()
-            for filename in config.get('jobs-from', {}))
-        for name, job in itertools.chain(jobs, jobs_from):
+        for name, job in config.get('jobs', {}).iteritems():
             if defaults:
                 job = merge(defaults, job)
+            job['job-from'] = 'kind.yml'
             yield name, job
 
+        for filename in config.get('jobs-from', []):
+            for name, job in load_yaml(path, filename).iteritems():
+                if defaults:
+                    job = merge(defaults, job)
+                job['job-from'] = filename
+                yield name, job
+
     for name, job in jobs():
         job['name'] = name
         logger.debug("Generating tasks for {} {}".format(kind, name))
         yield job
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -43,16 +43,17 @@ job_description_schema = Schema({
     Optional('name'): basestring,
     Optional('label'): basestring,
 
     # the following fields are passed directly through to the task description,
     # possibly modified by the run implementation.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details.
     Required('description'): task_description_schema['description'],
     Optional('attributes'): task_description_schema['attributes'],
+    Optional('job-from'): task_description_schema['job-from'],
     Optional('dependencies'): task_description_schema['dependencies'],
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -22,16 +22,17 @@ from voluptuous import Required
 transforms = TransformSequence()
 
 push_apk_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
     Required('name'): basestring,
     Required('label'): basestring,
     Required('description'): basestring,
+    Required('job-from'): basestring,
     Required('attributes'): object,
     Required('treeherder'): object,
     Required('run-on-projects'): list,
     Required('worker-type'): basestring,
     Required('worker'): object,
     Required('scopes'): None,
     Required('deadline-after'): basestring,
 })
--- a/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -20,16 +20,17 @@ from voluptuous import Required
 transforms = TransformSequence()
 
 push_apk_breakpoint_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
     Required('name'): basestring,
     Required('label'): basestring,
     Required('description'): basestring,
+    Required('job-from'): basestring,
     Required('attributes'): object,
     Required('worker-type'): None,
     Required('worker'): object,
     Required('treeherder'): object,
     Required('run-on-projects'): list,
     Required('deadline-after'): basestring,
 })
 
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -5,16 +5,17 @@
 Source-test jobs can run on multiple platforms.  These transforms allow jobs
 with either `platform` or a list of `platforms`, and set the appropriate
 treeherder configuration and attributes for that platform.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import os
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job import job_description_schema
 from taskgraph.util.attributes import keymatch
 from taskgraph.util.schema import (
     validate_schema,
     resolve_keyed_by,
 )
@@ -63,19 +64,23 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         yield validate_schema(source_test_description_schema, job,
                               "In job {!r}:".format(job['name']))
 
 
 @transforms.add
-def set_job_try_name(config, jobs):
+def set_job_name(config, jobs):
     for job in jobs:
         job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
+
+        if 'job-from' in job and job['job-from'] != 'kind.yml':
+            from_name = os.path.splitext(job['job-from'])[0]
+            job['name'] = '{}-{}'.format(from_name, job['name'])
         yield job
 
 
 @transforms.add
 def expand_platforms(config, jobs):
     for job in jobs:
         if isinstance(job['platform'], basestring):
             yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -50,16 +50,19 @@ task_description_schema = Schema({
     Required('label'): basestring,
 
     # description of the task (for metadata)
     Required('description'): basestring,
 
     # attributes for this task
     Optional('attributes'): {basestring: object},
 
+    # relative path (from config.path) to the file task was defined in
+    Optional('job-from'): basestring,
+
     # dependencies of this task, keyed by name; these are passed through
     # verbatim and subject to the interpretation of the Task's get_dependencies
     # method.
     Optional('dependencies'): {basestring: object},
 
     # expiration and deadline times, relative to task creation, with units
     # (e.g., "14 days").  Defaults are set based on the project.
     Optional('expires-after'): basestring,