Bug 1333255: handle when.files-changed at the job level; r=jonasfj draft
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 09 Mar 2017 15:16:25 -0500
changeset 498383 83e12588431d82260eccc6356207dabd4c85bae6
parent 498382 93a206dc9df029c23514f11d75e349d897dd44bf
child 498384 549773e05e18371a399612d9bceccffc29be8cf2
push id49162
push userdmitchell@mozilla.com
push dateTue, 14 Mar 2017 16:53:56 +0000
reviewersjonasfj
bugs1333255
milestone55.0a1
Bug 1333255: handle when.files-changed at the job level; r=jonasfj MozReview-Commit-ID: AbJcXu22dNP
taskcluster/ci/android-stuff/kind.yml
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -53,19 +53,19 @@ jobs:
             command:
               - "/bin/bash"
               - "-c"
               - "/home/worker/bin/before.sh && /home/worker/bin/build.sh && /home/worker/bin/after.sh && true\n"
             max-run-time: 36000
         scopes:
           - docker-worker:relengapi-proxy:tooltool.download.internal
           - docker-worker:relengapi-proxy:tooltool.download.public
-        when:
-            files-changed:
-              - "mobile/android/config/**"
+        optimizations:
+          - - files-changed
+            - - "mobile/android/config/**"
               - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
               - "**/*.gradle"
 
     android-test:
         description: "Android armv7 unit tests"
         attributes:
             build_platform: android-test
             build_type: opt
@@ -103,19 +103,19 @@ jobs:
             command:
               # NOTE: this could probably be a job description with run.using = 'mozharness'
               - "/bin/bash"
               - "bin/build.sh"
             max-run-time: 36000
         scopes:
           - docker-worker:relengapi-proxy:tooltool.download.internal
           - docker-worker:relengapi-proxy:tooltool.download.public
-        when:
-            files-changed:
-              - "mobile/android/base/**"
+        optimizations:
+          - - files-changed
+            - - "mobile/android/base/**"
               - "mobile/android/tests/background/junit4/**"
 
     android-lint:
         description: "Android lint"
         attributes:
             build_platform: android-lint
             build_type: opt
         treeherder:
@@ -158,19 +158,19 @@ jobs:
             command:
               # NOTE: this could probably be a job description with run.using = 'mozharness'
               - "/bin/bash"
               - "bin/build.sh"
             max-run-time: 36000
         scopes:
           - docker-worker:relengapi-proxy:tooltool.download.internal
           - docker-worker:relengapi-proxy:tooltool.download.public
-        when:
-            files-changed:
-              - "mobile/android/**/*.java"
+        optimizations:
+          - - files-changed
+            - - "mobile/android/**/*.java"
               - "mobile/android/**/*.jpeg"
               - "mobile/android/**/*.jpg"
               - "mobile/android/**/*.png"
               - "mobile/android/**/*.svg"
               - "mobile/android/**/*.xml" # Manifest & android resources
               - "mobile/android/**/build.gradle"
 
     android-checkstyle:
@@ -212,19 +212,19 @@ jobs:
             command:
               # NOTE: this could probably be a job description with run.using = 'mozharness'
               - "/bin/bash"
               - "bin/build.sh"
             max-run-time: 36000
         scopes:
           - docker-worker:relengapi-proxy:tooltool.download.internal
           - docker-worker:relengapi-proxy:tooltool.download.public
-        when:
-            files-changed:
-              - "mobile/android/**/checkstyle.xml"
+        optimizations:
+          - - files-changed
+            - - "mobile/android/**/checkstyle.xml"
               - "mobile/android/**/*.gradle"
               - "mobile/android/**/*.java"
 
     android-findbugs:
         description: "Android findbugs"
         attributes:
             build_platform: android-findbugs
             build_type: opt
@@ -262,12 +262,12 @@ jobs:
             command:
               # NOTE: this could probably be a job description with run.using = 'mozharness'
               - "/bin/bash"
               - "bin/build.sh"
             max-run-time: 36000
         scopes:
           - docker-worker:relengapi-proxy:tooltool.download.internal
           - docker-worker:relengapi-proxy:tooltool.download.public
-        when:
-            files-changed:
-              - "mobile/android/**/*.gradle"
+        optimizations:
+          - - files-changed
+            - - "mobile/android/**/*.gradle"
               - "mobile/android/**/*.java"
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -52,17 +52,26 @@ job_description_schema = Schema({
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('coalesce-name'): task_description_schema['coalesce-name'],
     Optional('optimizations'): task_description_schema['optimizations'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
-    Optional('when'): task_description_schema['when'],
+
+    # The "when" section contains descriptions of the circumstances
+    # under which this task should be included in the task graph.  This
+    # will be converted into an element in the `optimizations` list.
+    Optional('when'): Any({
+        # This task only needs to be run if a file matching one of the given
+        # patterns has changed in the push.  The patterns use the mozpack
+        # match function (python/mozbuild/mozpack/path.py).
+        Optional('files-changed'): [basestring],
+    }),
 
     # A description of how to run this job.
     'run': {
         # The key to a job implementation in a peer module to this one
         'using': basestring,
 
         # Any remaining content is verified against that job implementation's
         # own schema.
@@ -85,16 +94,40 @@ transforms = TransformSequence()
 @transforms.add
 def validate(config, jobs):
     for job in jobs:
         yield validate_schema(job_description_schema, job,
                               "In job {!r}:".format(job['name']))
 
 
 @transforms.add
+def rewrite_when_to_optimization(config, jobs):
+    for job in jobs:
+        when = job.pop('when', {})
+        files_changed = when.get('files-changed')
+        if not files_changed:
+            yield job
+            continue
+
+        # add some common files
+        files_changed.extend([
+            '{}/**'.format(config.path),
+            'taskcluster/taskgraph/**',
+        ])
+        if 'in-tree' in job['worker'].get('docker-image', {}):
+            files_changed.append('taskcluster/docker/{}/**'.format(
+                job['worker']['docker-image']['in-tree']))
+
+        job.setdefault('optimizations', []).append(['files-changed', files_changed])
+
+        assert 'when' not in job
+        yield job
+
+
+@transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
     import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -340,26 +340,16 @@ task_description_schema = Schema({
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
     }),
-
-    # The "when" section contains descriptions of the circumstances
-    # under which this task should be included in the task graph.  This
-    # will be converted into an element in the `optimizations` list.
-    Optional('when'): Any({
-        # This task only needs to be run if a file matching one of the given
-        # patterns has changed in the push.  The patterns use the mozpack
-        # match function (python/mozbuild/mozpack/path.py).
-        Optional('files-changed'): [basestring],
-    }),
 })
 
 GROUP_NAMES = {
     'py': 'Python unit tests',
     'tc': 'Executed by TaskCluster',
     'tc-e10s': 'Executed by TaskCluster with e10s',
     'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
     'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
@@ -768,44 +758,16 @@ def add_index_routes(config, tasks):
         else:
             extra_index['rank'] = rank
 
         del task['index']
         yield task
 
 
 @transforms.add
-def add_files_changed(config, tasks):
-    for task in tasks:
-        if 'files-changed' not in task.get('when', {}):
-            yield task
-            continue
-
-        task['when']['files-changed'].extend([
-            '{}/**'.format(config.path),
-            'taskcluster/taskgraph/**',
-        ])
-
-        if 'in-tree' in task['worker'].get('docker-image', {}):
-            task['when']['files-changed'].append('taskcluster/docker/{}/**'.format(
-                task['worker']['docker-image']['in-tree']))
-
-        yield task
-
-
-@transforms.add
-def setup_optimizations(config, tasks):
-    for task in tasks:
-        optimizations = task.setdefault('optimizations', [])
-        if 'when' in task and 'files-changed' in task['when']:
-            optimizations.append(['files-changed', task['when']['files-changed']])
-        yield task
-
-
-@transforms.add
 def build_task(config, tasks):
     for task in tasks:
         worker_type = task['worker-type'].format(level=str(config.params['level']))
         provisioner_id, worker_type = worker_type.split('/', 1)
 
         routes = task.get('routes', [])
         scopes = task.get('scopes', [])
 
@@ -888,17 +850,17 @@ def build_task(config, tasks):
         attributes = task.get('attributes', {})
         attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
 
         yield {
             'label': task['label'],
             'task': task_def,
             'dependencies': task.get('dependencies', {}),
             'attributes': attributes,
-            'optimizations': task['optimizations'],
+            'optimizations': task.get('optimizations', []),
         }
 
 
 # Check that the v2 route templates match those used by Mozharness.  This can
 # go away once Mozharness builds are no longer performed in Buildbot, and the
 # Mozharness code referencing routes.json is deleted.
 def check_v2_routes():
     with open("testing/mozharness/configs/routes.json", "rb") as f: