Bug 1249091 - Incorporate moz.build Files metadata into TaskCluster scheduling; r?garndt draft
authorGregory Szorc <gps@mozilla.com>
Wed, 17 Feb 2016 14:27:53 -0800
changeset 331695 5fdc0244e2e9363fbcac80eb604c84adcdf2f75f
parent 331694 c559632cdaa2b24252bfc0380c7ee5ad0bc1d2e8
child 331698 001bafcb55e98c26b4930940d7220f0248d81e38
push id11040
push usergszorc@mozilla.com
push dateWed, 17 Feb 2016 22:51:37 +0000
reviewersgarndt
bugs1249091
milestone47.0a1
Bug 1249091 - Incorporate moz.build Files metadata into TaskCluster scheduling; r?garndt Querying moz.build files info from Python is pretty simple: just instantiate a BuildReader and pass it a list of paths. We already have the (possibly empty) set of changed files from version control. So querying against the moz.build data is pretty easy! As part of this, the eslint-gecko file patterns have been moved into moz.build files. The moz.build syntax is a bit verbose because Files() are limited to a single pattern. There is an open bug on supporting multiple patterns per Files() instance that will make this cleaner. MozReview-Commit-ID: 3htQK9oDGoF
moz.build
testing/moz.build
testing/taskcluster/mach_commands.py
testing/taskcluster/taskcluster_graph/commit_parser.py
testing/taskcluster/tasks/branches/base_jobs.yml
--- a/moz.build
+++ b/moz.build
@@ -3,16 +3,39 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files('**/Makefile.in'):
     BUG_COMPONENT = ('Core', 'Build Config')
     FINAL = True
 
+# Schedule eslint whenever linted files change.
+with Files('**/*.js'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.jsm'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.jsx'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.html'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+# And when the lint configuration files change.
+with Files('**/.eslintignore'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*eslintrc*'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('tools/lint/**'):
+    IMPACTED_TASKS += ['eslint-gecko']
+
 FILES_PER_UNIFIED_FILE = 1
 
 CONFIGURE_SUBST_FILES += [
     'config/autoconf.mk',
     'config/emptyvars.mk',
 ]
 
 if CONFIG['ENABLE_CLANG_PLUGIN']:
--- a/testing/moz.build
+++ b/testing/moz.build
@@ -1,7 +1,11 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-SPHINX_TREES['taskcluster'] = 'taskcluster/docs'
\ No newline at end of file
+SPHINX_TREES['taskcluster'] = 'taskcluster/docs'
+
+# Always run eslint after the plugin changes.
+with Files('eslint-plugin-mozilla/**'):
+    IMPACTED_TASKS += ['eslint-gecko']
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -16,16 +16,20 @@ import time
 from collections import namedtuple
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
+from mozbuild.base import (
+    MachCommandBase,
+)
+
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
 
 # XXX: If/when we have the taskcluster queue use construct url instead
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
@@ -222,17 +226,17 @@ class DecisionTask(object):
             'as_slugid': SlugidJar(),
             'from_now': json_time_from_now,
             'now': current_json_time()
         }.items())
         task = templates.load(params['task'], parameters)
         print(json.dumps(task, indent=4))
 
 @CommandProvider
-class Graph(object):
+class Graph(MachCommandBase):
     @Command('taskcluster-graph', category="ci",
         description="Create taskcluster task graph")
     @CommandArgument('--base-repository',
         default=os.environ.get('GECKO_BASE_REPOSITORY'),
         help='URL for "base" repository to clone')
     @CommandArgument('--head-repository',
         default=os.environ.get('GECKO_HEAD_REPOSITORY'),
         help='URL for "head" repository to fetch revision from')
@@ -275,16 +279,26 @@ class Graph(object):
         action='store_true', default=False,
         help="Stub out taskIds and date fields from the task definitions.")
     @CommandArgument('--ignore-conditions',
         action='store_true',
         help='Run tasks even if their conditions are not met')
     def create_graph(self, **params):
         from functools import partial
 
+        from mozbuild.frontend.context import (
+            Files,
+        )
+        from mozbuild.frontend.reader import (
+            BuildReader,
+            EmptyConfig,
+        )
+        from mozpack.files import (
+            FileFinder,
+        )
         from mozpack.path import match as mozpackmatch
 
         from slugid import nice as slugid
 
         import taskcluster_graph.transform.routes as routes_transform
         from taskcluster_graph.commit_parser import parse_commit
         from taskcluster_graph.image_builder import (
             docker_image,
@@ -394,29 +408,53 @@ class Graph(object):
             'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
             'description': 'Task graph generated via ./mach taskcluster-graph',
             'name': 'task graph local'
         }
 
         # Filter the job graph according to conditions met by this invocation run.
+
+        # Obtain moz.build files info.
+        build_config = EmptyConfig(self.topsrcdir)
+        finder = FileFinder(self.topsrcdir, find_executables=False)
+        reader = BuildReader(build_config, finder=finder)
+        mozbuild_files_info = reader.files_info(changed_files)
+        aggregate_files_info = Files.aggregate(mozbuild_files_info)
+        mozbuild_impacted_tasks = set(aggregate_files_info['impacted_tasks'])
+
         def should_run(task):
             # Command line override to not filter.
             if params['ignore_conditions']:
                 return True
 
             when = task['when']
 
             if when.get('always', False):
                 return True
 
             if task.get('explicitly-added', False):
                 return True
 
+            # If moz.build says the task is impacted, schedule it.
+            tags = task.get('tags', [])
+            if 'name' in task:
+                if task['name'] in mozbuild_impacted_tasks:
+                    sys.stderr.write('scheduling %s because moz.build said it was '
+                                     'impacted\n' % task['name'])
+                    return True
+
+                for tag in tags:
+                    if tag in mozbuild_impacted_tasks:
+                        sys.stderr.write('scheduling %s because moz.build said '
+                                         'tag %s was impacted\n' % (
+                                         task['name'], tag))
+                        return True
+
             # If the task defines file patterns and we have a set of changed
             # files to compare against, only run if a file pattern matches one
             # of the changed files.
             file_patterns = when.get('file_patterns', None)
             if file_patterns and changed_files:
                 for pattern in file_patterns:
                     for path in changed_files:
                         if mozpackmatch(path, pattern):
--- a/testing/taskcluster/taskcluster_graph/commit_parser.py
+++ b/testing/taskcluster/taskcluster_graph/commit_parser.py
@@ -355,18 +355,20 @@ def parse_commit(message, jobs):
         if not run:
             continue
 
         # TODO support tasks that are defined as dependent on another one.
         if not task.get('root', False):
             continue
 
         result.append({
+            'name': name,
             'task': task['task'],
             'explicitly-added': explicit,
+            'tags': task.get('tags', []),
             'post-build': [],
             'dependents': [],
             'additional-parameters': task.get('additional-parameters', {}),
             'build_name': name,
             # TODO support declaring a different build type
             'build_type': name,
             'interactive': args.interactive,
             'when': task.get('when', {})
--- a/testing/taskcluster/tasks/branches/base_jobs.yml
+++ b/testing/taskcluster/tasks/branches/base_jobs.yml
@@ -305,23 +305,9 @@ tests:
       tasks/builds/dbg_linux64.yml:
         task: tasks/tests/fx_linux64_xpcshell.yml
 
 # Miscellaneous tasks.
 tasks:
   eslint-gecko:
     task: tasks/tests/eslint-gecko.yml
     root: true
-    when:
-      file_patterns:
-        # Files that are likely audited.
-        - '**/*.js'
-        - '**/*.jsm'
-        - '**/*.jsx'
-        - '**/*.html'
-        - '**/*.xml'
-        # Run when eslint policies change.
-        - '**/.eslintignore'
-        - '**/*eslintrc*'
-        # The plugin implementing custom checks.
-        - 'testing/eslint-plugin-mozilla/**'
-        # Other misc lint related files.
-        - 'tools/lint/**'
+