Bug 1333255: handle when.files-changed at the job level; r=jonasfj
MozReview-Commit-ID: AbJcXu22dNP
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -53,19 +53,19 @@ jobs:
command:
- "/bin/bash"
- "-c"
- "/home/worker/bin/before.sh && /home/worker/bin/build.sh && /home/worker/bin/after.sh && true\n"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
- when:
- files-changed:
- - "mobile/android/config/**"
+ optimizations:
+ - - files-changed
+ - - "mobile/android/config/**"
- "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
- "**/*.gradle"
android-test:
description: "Android armv7 unit tests"
attributes:
build_platform: android-test
build_type: opt
@@ -103,19 +103,19 @@ jobs:
command:
# NOTE: this could probably be a job description with run.using = 'mozharness'
- "/bin/bash"
- "bin/build.sh"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
- when:
- files-changed:
- - "mobile/android/base/**"
+ optimizations:
+ - - files-changed
+ - - "mobile/android/base/**"
- "mobile/android/tests/background/junit4/**"
android-lint:
description: "Android lint"
attributes:
build_platform: android-lint
build_type: opt
treeherder:
@@ -158,19 +158,19 @@ jobs:
command:
# NOTE: this could probably be a job description with run.using = 'mozharness'
- "/bin/bash"
- "bin/build.sh"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
- when:
- files-changed:
- - "mobile/android/**/*.java"
+ optimizations:
+ - - files-changed
+ - - "mobile/android/**/*.java"
- "mobile/android/**/*.jpeg"
- "mobile/android/**/*.jpg"
- "mobile/android/**/*.png"
- "mobile/android/**/*.svg"
- "mobile/android/**/*.xml" # Manifest & android resources
- "mobile/android/**/build.gradle"
android-checkstyle:
@@ -212,19 +212,19 @@ jobs:
command:
# NOTE: this could probably be a job description with run.using = 'mozharness'
- "/bin/bash"
- "bin/build.sh"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
- when:
- files-changed:
- - "mobile/android/**/checkstyle.xml"
+ optimizations:
+ - - files-changed
+ - - "mobile/android/**/checkstyle.xml"
- "mobile/android/**/*.gradle"
- "mobile/android/**/*.java"
android-findbugs:
description: "Android findbugs"
attributes:
build_platform: android-findbugs
build_type: opt
@@ -262,12 +262,12 @@ jobs:
command:
# NOTE: this could probably be a job description with run.using = 'mozharness'
- "/bin/bash"
- "bin/build.sh"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
- when:
- files-changed:
- - "mobile/android/**/*.gradle"
+ optimizations:
+ - - files-changed
+ - - "mobile/android/**/*.gradle"
- "mobile/android/**/*.java"
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -52,17 +52,26 @@ job_description_schema = Schema({
Optional('tags'): task_description_schema['tags'],
Optional('extra'): task_description_schema['extra'],
Optional('treeherder'): task_description_schema['treeherder'],
Optional('index'): task_description_schema['index'],
Optional('run-on-projects'): task_description_schema['run-on-projects'],
Optional('coalesce-name'): task_description_schema['coalesce-name'],
Optional('optimizations'): task_description_schema['optimizations'],
Optional('needs-sccache'): task_description_schema['needs-sccache'],
- Optional('when'): task_description_schema['when'],
+
+ # The "when" section contains descriptions of the circumstances
+ # under which this task should be included in the task graph. This
+ # will be converted into an element in the `optimizations` list.
+ Optional('when'): Any({
+ # This task only needs to be run if a file matching one of the given
+ # patterns has changed in the push. The patterns use the mozpack
+ # match function (python/mozbuild/mozpack/path.py).
+ Optional('files-changed'): [basestring],
+ }),
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
@@ -85,16 +94,40 @@ transforms = TransformSequence()
@transforms.add
def validate(config, jobs):
for job in jobs:
yield validate_schema(job_description_schema, job,
"In job {!r}:".format(job['name']))
@transforms.add
+def rewrite_when_to_optimization(config, jobs):
+ for job in jobs:
+ when = job.pop('when', {})
+ files_changed = when.get('files-changed')
+ if not files_changed:
+ yield job
+ continue
+
+ # add some common files
+ files_changed.extend([
+ '{}/**'.format(config.path),
+ 'taskcluster/taskgraph/**',
+ ])
+ if 'in-tree' in job['worker'].get('docker-image', {}):
+ files_changed.append('taskcluster/docker/{}/**'.format(
+ job['worker']['docker-image']['in-tree']))
+
+ job.setdefault('optimizations', []).append(['files-changed', files_changed])
+
+ assert 'when' not in job
+ yield job
+
+
+@transforms.add
def make_task_description(config, jobs):
"""Given a build description, create a task description"""
# import plugin modules first, before iterating over jobs
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -340,26 +340,16 @@ task_description_schema = Schema({
# type of signing task (for CoT)
Required('taskType'): basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
}],
}),
-
- # The "when" section contains descriptions of the circumstances
- # under which this task should be included in the task graph. This
- # will be converted into an element in the `optimizations` list.
- Optional('when'): Any({
- # This task only needs to be run if a file matching one of the given
- # patterns has changed in the push. The patterns use the mozpack
- # match function (python/mozbuild/mozpack/path.py).
- Optional('files-changed'): [basestring],
- }),
})
GROUP_NAMES = {
'py': 'Python unit tests',
'tc': 'Executed by TaskCluster',
'tc-e10s': 'Executed by TaskCluster with e10s',
'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
@@ -768,44 +758,16 @@ def add_index_routes(config, tasks):
else:
extra_index['rank'] = rank
del task['index']
yield task
@transforms.add
-def add_files_changed(config, tasks):
- for task in tasks:
- if 'files-changed' not in task.get('when', {}):
- yield task
- continue
-
- task['when']['files-changed'].extend([
- '{}/**'.format(config.path),
- 'taskcluster/taskgraph/**',
- ])
-
- if 'in-tree' in task['worker'].get('docker-image', {}):
- task['when']['files-changed'].append('taskcluster/docker/{}/**'.format(
- task['worker']['docker-image']['in-tree']))
-
- yield task
-
-
-@transforms.add
-def setup_optimizations(config, tasks):
- for task in tasks:
- optimizations = task.setdefault('optimizations', [])
- if 'when' in task and 'files-changed' in task['when']:
- optimizations.append(['files-changed', task['when']['files-changed']])
- yield task
-
-
-@transforms.add
def build_task(config, tasks):
for task in tasks:
worker_type = task['worker-type'].format(level=str(config.params['level']))
provisioner_id, worker_type = worker_type.split('/', 1)
routes = task.get('routes', [])
scopes = task.get('scopes', [])
@@ -888,17 +850,17 @@ def build_task(config, tasks):
attributes = task.get('attributes', {})
attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
yield {
'label': task['label'],
'task': task_def,
'dependencies': task.get('dependencies', {}),
'attributes': attributes,
- 'optimizations': task['optimizations'],
+ 'optimizations': task.get('optimizations', []),
}
# Check that the v2 route templates match those used by Mozharness. This can
# go away once Mozharness builds are no longer performed in Buildbot, and the
# Mozharness code referencing routes.json is deleted.
def check_v2_routes():
with open("testing/mozharness/configs/routes.json", "rb") as f: