Bug 1249091 - Incorporate moz.build Files metadata into TaskCluster scheduling; r?garndt
Querying moz.build files info from Python is pretty simple: just
instantiate a BuildReader and pass it a list of paths. We already have
the (possibly empty) set of changed files from version control. So
querying against the moz.build data is pretty easy!
As part of this, the eslint-gecko file patterns have been moved into
moz.build files. The moz.build syntax is a bit verbose because Files()
are limited to a single pattern. There is an open bug on supporting
multiple patterns per Files() instance that will make this cleaner.
MozReview-Commit-ID: 3htQK9oDGoF
--- a/moz.build
+++ b/moz.build
@@ -3,16 +3,39 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
with Files('**/Makefile.in'):
BUG_COMPONENT = ('Core', 'Build Config')
FINAL = True
+# Schedule eslint whenever linted files change.
+with Files('**/*.js'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.jsm'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.jsx'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*.html'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+# And when the lint configuration files change.
+with Files('**/.eslintignore'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('**/*eslintrc*'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
+with Files('tools/lint/**'):
+ IMPACTED_TASKS += ['eslint-gecko']
+
FILES_PER_UNIFIED_FILE = 1
CONFIGURE_SUBST_FILES += [
'config/autoconf.mk',
'config/emptyvars.mk',
]
if CONFIG['ENABLE_CLANG_PLUGIN']:
--- a/testing/moz.build
+++ b/testing/moz.build
@@ -1,7 +1,11 @@
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-SPHINX_TREES['taskcluster'] = 'taskcluster/docs'
\ No newline at end of file
+SPHINX_TREES['taskcluster'] = 'taskcluster/docs'
+
+# Always run eslint after the plugin changes.
+with Files('eslint-plugin-mozilla/**'):
+ IMPACTED_TASKS += ['eslint-gecko']
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -16,16 +16,20 @@ import time
from collections import namedtuple
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
+from mozbuild.base import (
+ MachCommandBase,
+)
+
ROOT = os.path.dirname(os.path.realpath(__file__))
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
# XXX: If/when we have the taskcluster queue use construct url instead
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
@@ -222,17 +226,17 @@ class DecisionTask(object):
'as_slugid': SlugidJar(),
'from_now': json_time_from_now,
'now': current_json_time()
}.items())
task = templates.load(params['task'], parameters)
print(json.dumps(task, indent=4))
@CommandProvider
-class Graph(object):
+class Graph(MachCommandBase):
@Command('taskcluster-graph', category="ci",
description="Create taskcluster task graph")
@CommandArgument('--base-repository',
default=os.environ.get('GECKO_BASE_REPOSITORY'),
help='URL for "base" repository to clone')
@CommandArgument('--head-repository',
default=os.environ.get('GECKO_HEAD_REPOSITORY'),
help='URL for "head" repository to fetch revision from')
@@ -275,16 +279,26 @@ class Graph(object):
action='store_true', default=False,
help="Stub out taskIds and date fields from the task definitions.")
@CommandArgument('--ignore-conditions',
action='store_true',
help='Run tasks even if their conditions are not met')
def create_graph(self, **params):
from functools import partial
+ from mozbuild.frontend.context import (
+ Files,
+ )
+ from mozbuild.frontend.reader import (
+ BuildReader,
+ EmptyConfig,
+ )
+ from mozpack.files import (
+ FileFinder,
+ )
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
import taskcluster_graph.transform.routes as routes_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (
docker_image,
@@ -394,29 +408,53 @@ class Graph(object):
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
# Filter the job graph according to conditions met by this invocation run.
+
+ # Obtain moz.build files info.
+ build_config = EmptyConfig(self.topsrcdir)
+ finder = FileFinder(self.topsrcdir, find_executables=False)
+ reader = BuildReader(build_config, finder=finder)
+ mozbuild_files_info = reader.files_info(changed_files)
+ aggregate_files_info = Files.aggregate(mozbuild_files_info)
+ mozbuild_impacted_tasks = set(aggregate_files_info['impacted_tasks'])
+
def should_run(task):
# Command line override to not filter.
if params['ignore_conditions']:
return True
when = task['when']
if when.get('always', False):
return True
if task.get('explicitly-added', False):
return True
+ # If moz.build says the task is impacted, schedule it.
+ tags = task.get('tags', [])
+ if 'name' in task:
+ if task['name'] in mozbuild_impacted_tasks:
+ sys.stderr.write('scheduling %s because moz.build said it was '
+ 'impacted\n' % task['name'])
+ return True
+
+ for tag in tags:
+ if tag in mozbuild_impacted_tasks:
+ sys.stderr.write('scheduling %s because moz.build said '
+ 'tag %s was impacted\n' % (
+ task['name'], tag))
+ return True
+
# If the task defines file patterns and we have a set of changed
# files to compare against, only run if a file pattern matches one
# of the changed files.
file_patterns = when.get('file_patterns', None)
if file_patterns and changed_files:
for pattern in file_patterns:
for path in changed_files:
if mozpackmatch(path, pattern):
--- a/testing/taskcluster/taskcluster_graph/commit_parser.py
+++ b/testing/taskcluster/taskcluster_graph/commit_parser.py
@@ -355,18 +355,20 @@ def parse_commit(message, jobs):
if not run:
continue
# TODO support tasks that are defined as dependent on another one.
if not task.get('root', False):
continue
result.append({
+ 'name': name,
'task': task['task'],
'explicitly-added': explicit,
+ 'tags': task.get('tags', []),
'post-build': [],
'dependents': [],
'additional-parameters': task.get('additional-parameters', {}),
'build_name': name,
# TODO support declaring a different build type
'build_type': name,
'interactive': args.interactive,
'when': task.get('when', {})
--- a/testing/taskcluster/tasks/branches/base_jobs.yml
+++ b/testing/taskcluster/tasks/branches/base_jobs.yml
@@ -305,23 +305,9 @@ tests:
tasks/builds/dbg_linux64.yml:
task: tasks/tests/fx_linux64_xpcshell.yml
# Miscellaneous tasks.
tasks:
eslint-gecko:
task: tasks/tests/eslint-gecko.yml
root: true
- when:
- file_patterns:
- # Files that are likely audited.
- - '**/*.js'
- - '**/*.jsm'
- - '**/*.jsx'
- - '**/*.html'
- - '**/*.xml'
- # Run when eslint policies change.
- - '**/.eslintignore'
- - '**/*eslintrc*'
- # The plugin implementing custom checks.
- - 'testing/eslint-plugin-mozilla/**'
- # Other misc lint related files.
- - 'tools/lint/**'
+