Bug 1395717 - Use filename source-test tasks are defined in as part of their label, r?dustin
This creates a new "job-from" field that contains the relative filename the job was defined
in. The filename is relative to 'config.path'. If the task came from the 'jobs' key defined
in kind.yml, this field will be set to 'kind.yml'.
MozReview-Commit-ID: 9e1tEb6XuZT
--- a/taskcluster/ci/source-test/doc.yml
+++ b/taskcluster/ci/source-test/doc.yml
@@ -1,9 +1,9 @@
-doc-generate:
+generate:
description: Generate the Sphinx documentation
platform: lint/opt
treeherder:
symbol: tc(Doc)
kind: test
tier: 1
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
worker:
@@ -19,17 +19,17 @@ doc-generate:
cd /builds/worker/checkouts/gecko &&
./mach doc --outdir docs-out --no-open --archive
when:
files-changed:
- '**/*.py'
- '**/*.rst'
- 'tools/docs/**'
-doc-upload:
+upload:
description: Generate and upload the Sphinx documentation
platform: lint/opt
treeherder:
symbol: tc(DocUp)
kind: test
tier: 3
run-on-projects: [mozilla-central]
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -9,17 +9,17 @@ transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
jobs-from:
- cram.yml
- doc.yml
- mocha.yml
- mozlint.yml
- - python-tests.yml
+ - python.yml
- webidl.yml
# This is used by run-task based tasks to lookup which build task it
# should depend on based on its own platform.
dependent-build-platforms:
linux64-asan/opt:
label: build-linux64-asan/opt
target-name: target.tar.bz2
--- a/taskcluster/ci/source-test/mozlint.yml
+++ b/taskcluster/ci/source-test/mozlint.yml
@@ -1,9 +1,9 @@
-mozlint-eslint:
+eslint:
description: JS lint check
platform: lint/opt
treeherder:
symbol: ES
kind: test
tier: 1
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
worker:
@@ -31,17 +31,37 @@ mozlint-eslint:
- '**/*eslintrc*'
# The plugin implementing custom checks.
- 'tools/lint/eslint/eslint-plugin-mozilla/**'
- 'tools/lint/eslint/eslint-plugin-spidermonkey-js/**'
# Other misc lint related files.
- 'python/mozlint/**'
- 'tools/lint/**'
-mozlint-py-flake8:
+py-compat:
+ description: lint for python 2/3 compatibility issues
+ platform: lint/opt
+ treeherder:
+ symbol: py-compat
+ kind: test
+ tier: 1
+ worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+ worker:
+ docker-image: {in-tree: "lint"}
+ max-run-time: 1800
+ run:
+ using: mach
+ mach: lint -l py2 -l py3 -f treeherder
+ when:
+ files-changed:
+ - '**/*.py'
+ - 'python/mozlint/**'
+ - 'tools/lint/**'
+
+py-flake8:
description: flake8 run over the gecko codebase
platform: lint/opt
treeherder:
symbol: f8
kind: test
tier: 1
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
worker:
@@ -52,58 +72,16 @@ mozlint-py-flake8:
mach: lint -l flake8 -f treeherder
when:
files-changed:
- '**/*.py'
- '**/.flake8'
- 'python/mozlint/**'
- 'tools/lint/**'
-mozlint-yaml:
- description: yamllint run over the gecko codebase
- platform: lint/opt
- treeherder:
- symbol: yaml
- kind: test
- tier: 1
- worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
- worker:
- docker-image: {in-tree: "lint"}
- max-run-time: 1800
- run:
- using: mach
- mach: lint -l yaml -f treeherder
- when:
- files-changed:
- - '**/*.yml'
- - '**/*.yaml'
- - '**/.ymllint'
- - 'python/mozlint/**'
- - 'tools/lint/**'
-
-mozlint-py-compat:
- description: lint for python 2/3 compatibility issues
- platform: lint/opt
- treeherder:
- symbol: py-compat
- kind: test
- tier: 1
- worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
- worker:
- docker-image: {in-tree: "lint"}
- max-run-time: 1800
- run:
- using: mach
- mach: lint -l py2 -l py3 -f treeherder
- when:
- files-changed:
- - '**/*.py'
- - 'python/mozlint/**'
- - 'tools/lint/**'
-
wptlint-gecko:
description: web-platform-tests linter
platform: lint/opt
treeherder:
symbol: W
kind: test
tier: 1
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
@@ -116,8 +94,30 @@ wptlint-gecko:
when:
files-changed:
- 'testing/web-platform/tests/**'
- 'testing/web-platform/mozilla/tests/**'
- 'testing/web-platform/meta/MANIFEST.json'
- 'testing/web-platform/mozilla/meta/MANIFEST.json'
- 'python/mozlint/**'
- 'tools/lint/**'
+
+yaml:
+ description: yamllint run over the gecko codebase
+ platform: lint/opt
+ treeherder:
+ symbol: yaml
+ kind: test
+ tier: 1
+ worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+ worker:
+ docker-image: {in-tree: "lint"}
+ max-run-time: 1800
+ run:
+ using: mach
+ mach: lint -l yaml -f treeherder
+ when:
+ files-changed:
+ - '**/*.yml'
+ - '**/*.yaml'
+ - '**/.ymllint'
+ - 'python/mozlint/**'
+ - 'tools/lint/**'
rename from taskcluster/ci/source-test/python-tests.yml
rename to taskcluster/ci/source-test/python.yml
--- a/taskcluster/ci/source-test/webidl.yml
+++ b/taskcluster/ci/source-test/webidl.yml
@@ -1,9 +1,9 @@
-webidl-test:
+test:
description: WebIDL parser tests
platform: lint/opt
treeherder:
symbol: Wp
kind: test
tier: 1
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
worker:
--- a/taskcluster/taskgraph/loader/transform.py
+++ b/taskcluster/taskgraph/loader/transform.py
@@ -1,16 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
-import itertools
from ..util.templates import merge
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
def loader(kind, path, config, params, loaded_tasks):
@@ -28,21 +27,25 @@ def loader(kind, path, config, params, l
kind. More complex defaults should be implemented with custom
transforms.
Other kind implementations can use a different loader function to
produce inputs and hand them to `transform_inputs`.
"""
def jobs():
defaults = config.get('job-defaults')
- jobs = config.get('jobs', {}).iteritems()
- jobs_from = itertools.chain.from_iterable(
- load_yaml(path, filename).iteritems()
- for filename in config.get('jobs-from', {}))
- for name, job in itertools.chain(jobs, jobs_from):
+ for name, job in config.get('jobs', {}).iteritems():
if defaults:
job = merge(defaults, job)
+ job['job-from'] = 'kind.yml'
yield name, job
+ for filename in config.get('jobs-from', []):
+ for name, job in load_yaml(path, filename).iteritems():
+ if defaults:
+ job = merge(defaults, job)
+ job['job-from'] = filename
+ yield name, job
+
for name, job in jobs():
job['name'] = name
logger.debug("Generating tasks for {} {}".format(kind, name))
yield job
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -43,16 +43,17 @@ job_description_schema = Schema({
Optional('name'): basestring,
Optional('label'): basestring,
# the following fields are passed directly through to the task description,
# possibly modified by the run implementation. See
# taskcluster/taskgraph/transforms/task.py for the schema details.
Required('description'): task_description_schema['description'],
Optional('attributes'): task_description_schema['attributes'],
+ Optional('job-from'): task_description_schema['job-from'],
Optional('dependencies'): task_description_schema['dependencies'],
Optional('expires-after'): task_description_schema['expires-after'],
Optional('routes'): task_description_schema['routes'],
Optional('scopes'): task_description_schema['scopes'],
Optional('tags'): task_description_schema['tags'],
Optional('extra'): task_description_schema['extra'],
Optional('treeherder'): task_description_schema['treeherder'],
Optional('index'): task_description_schema['index'],
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@@ -22,16 +22,17 @@ from voluptuous import Required
transforms = TransformSequence()
push_apk_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): basestring,
Required('description'): basestring,
+ Required('job-from'): basestring,
Required('attributes'): object,
Required('treeherder'): object,
Required('run-on-projects'): list,
Required('worker-type'): basestring,
Required('worker'): object,
Required('scopes'): None,
Required('deadline-after'): basestring,
})
--- a/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
+++ b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py
@@ -20,16 +20,17 @@ from voluptuous import Required
transforms = TransformSequence()
push_apk_breakpoint_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): basestring,
Required('description'): basestring,
+ Required('job-from'): basestring,
Required('attributes'): object,
Required('worker-type'): None,
Required('worker'): object,
Required('treeherder'): object,
Required('run-on-projects'): list,
Required('deadline-after'): basestring,
})
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -5,16 +5,17 @@
Source-test jobs can run on multiple platforms. These transforms allow jobs
with either `platform` or a list of `platforms`, and set the appropriate
treeherder configuration and attributes for that platform.
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
+import os
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.job import job_description_schema
from taskgraph.util.attributes import keymatch
from taskgraph.util.schema import (
validate_schema,
resolve_keyed_by,
)
@@ -63,19 +64,23 @@ transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
yield validate_schema(source_test_description_schema, job,
"In job {!r}:".format(job['name']))
@transforms.add
-def set_job_try_name(config, jobs):
+def set_job_name(config, jobs):
for job in jobs:
job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
+
+ if 'job-from' in job and job['job-from'] != 'kind.yml':
+ from_name = os.path.splitext(job['job-from'])[0]
+ job['name'] = '{}-{}'.format(from_name, job['name'])
yield job
@transforms.add
def expand_platforms(config, jobs):
for job in jobs:
if isinstance(job['platform'], basestring):
yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -50,16 +50,19 @@ task_description_schema = Schema({
Required('label'): basestring,
# description of the task (for metadata)
Required('description'): basestring,
# attributes for this task
Optional('attributes'): {basestring: object},
+ # relative path (from config.path) to the file task was defined in
+ Optional('job-from'): basestring,
+
# dependencies of this task, keyed by name; these are passed through
# verbatim and subject to the interpretation of the Task's get_dependencies
# method.
Optional('dependencies'): {basestring: object},
# expiration and deadline times, relative to task creation, with units
# (e.g., "14 days"). Defaults are set based on the project.
Optional('expires-after'): basestring,