new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/files_changed.py
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Support for optimizing tasks based on the set of files that have changed.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import requests
+from redo import retry
+from mozpack.path import match as mozpackmatch
+
+logger = logging.getLogger(__name__)
+_cache = {}
+
+
+def get_changed_files(repository, revision):
+ """
+ Get the set of files changed in the push headed by the given revision.
+ Responses are cached, so multiple calls with the same arguments are OK.
+ """
+ key = repository, revision
+ if key not in _cache:
+ url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
+ logger.debug("Querying version control for metadata: %s", url)
+
+ def get_automationrelevance():
+ response = requests.get(url, timeout=5)
+ return response.json()
+ contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
+
+ logger.debug('{} commits influencing task scheduling:'
+ .format(len(contents['changesets'])))
+ changed_files = set()
+ for c in contents['changesets']:
+ logger.debug(" {cset} {desc}".format(
+ cset=c['node'][0:12],
+ desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+ changed_files |= set(c['files'])
+
+ _cache[key] = changed_files
+ return _cache[key]
+
+
+def check(params, file_patterns):
+ """Determine whether any of the files changed in the indicated push to
+ https://hg.mozilla.org match any of the given file patterns."""
+ repository = params.get('head_repository')
+ revision = params.get('head_rev')
+ if not repository or not revision:
+ logger.warning("Missing `head_repository` or `head_rev` parameters; "
+ "assuming all files have changed")
+ return True
+
+ changed_files = get_changed_files(repository, revision)
+
+ for pattern in file_patterns:
+ for path in changed_files:
+ if mozpackmatch(path, pattern):
+ return True
+
+ return False
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -198,16 +198,17 @@ class TaskGraphGenerator(object):
target_graph)
yield 'target_task_graph', target_task_graph
logger.info("Generating optimized task graph")
do_not_optimize = set()
if not self.parameters.get('optimize_target_tasks', True):
do_not_optimize = target_task_set.graph.nodes
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
+ self.parameters,
do_not_optimize)
yield 'label_to_taskid', label_to_taskid
yield 'optimized_task_graph', optimized_task_graph
def _run_until(self, name):
while name not in self._run_results:
try:
k, v = self._run.next()
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -9,30 +9,31 @@ import re
from .graph import Graph
from .taskgraph import TaskGraph
from slugid import nice as slugid
logger = logging.getLogger(__name__)
TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
-def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None):
+def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
"""
Perform task optimization, without optimizing tasks named in
do_not_optimize.
"""
named_links_dict = target_task_graph.graph.named_links_dict()
label_to_taskid = {}
# This proceeds in two phases. First, mark all optimized tasks (those
# which will be removed from the graph) as such, including a replacement
# taskId where applicable. Second, generate a new task graph containing
# only the non-optimized tasks, with all task labels resolved to taskIds
# and with task['dependencies'] populated.
annotate_task_graph(target_task_graph=target_task_graph,
+ params=params,
do_not_optimize=do_not_optimize,
named_links_dict=named_links_dict,
label_to_taskid=label_to_taskid,
existing_tasks=existing_tasks)
return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid
def resolve_task_references(label, task_def, taskid_for_edge_name):
@@ -54,17 +55,17 @@ def resolve_task_references(label, task_
return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference'])
else:
return {k: recurse(v) for k, v in val.iteritems()}
else:
return val
return recurse(task_def)
-def annotate_task_graph(target_task_graph, do_not_optimize,
+def annotate_task_graph(target_task_graph, params, do_not_optimize,
named_links_dict, label_to_taskid, existing_tasks):
"""
Annotate each task in the graph with .optimized (boolean) and .task_id
(possibly None), following the rules for optimization and calling the task
kinds' `optimize_task` method.
As a side effect, label_to_taskid is updated with labels for all optimized
tasks that are replaced with existing tasks.
@@ -89,17 +90,17 @@ def annotate_task_graph(target_task_grap
if label in do_not_optimize:
optimized = False
# Let's check whether this task has been created before
elif existing_tasks is not None and label in existing_tasks:
optimized = True
replacement_task_id = existing_tasks[label]
# otherwise, examine the task itself (which may be an expensive operation)
else:
- optimized, replacement_task_id = task.optimize()
+ optimized, replacement_task_id = task.optimize(params)
task.optimized = optimized
task.task_id = replacement_task_id
if replacement_task_id:
label_to_taskid[label] = replacement_task_id
if optimized:
if replacement_task_id:
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -74,17 +74,17 @@ class Task(object):
"""
Get the set of task labels this task depends on, by querying the full
task set, given as `taskgraph`.
Returns a list of (task_label, dependency_name) pairs describing the
dependencies.
"""
- def optimize(self):
+ def optimize(self, params):
"""
Determine whether this task can be optimized, and if it can, what taskId
it should be replaced with.
The return value is a tuple `(optimized, taskId)`. If `optimized` is
true, then the task will be optimized (in other words, not included in
the task graph). If the second argument is a taskid, then any
dependencies on this task will isntead depend on that taskId. It is an
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -113,17 +113,17 @@ class DockerImageTask(base.Task):
task=image_task['task'], attributes=attributes,
index_paths=index_paths))
return tasks
def get_dependencies(self, taskgraph):
return []
- def optimize(self):
+ def optimize(self, params):
for index_path in self.index_paths:
try:
url = INDEX_URL.format(index_path)
existing_task = json.load(urllib2.urlopen(url))
# Only return the task ID if the artifact exists for the indexed
# task. Otherwise, continue on looking at each of the branches. Method
# continues trying other branches in case mozilla-central has an expired
# artifact, but 'project' might not. Only return no task ID if all
--- a/taskcluster/taskgraph/task/legacy.py
+++ b/taskcluster/taskgraph/task/legacy.py
@@ -613,17 +613,17 @@ class LegacyTask(base.Task):
# add a dependency on an image task, if needed
if 'docker-image' in self.task_dict:
deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
'docker-image'))
return deps
- def optimize(self):
+ def optimize(self, params):
# no optimization for the moment
return False, None
@classmethod
def from_json(cls, task_dict):
legacy_task = cls(kind='legacy',
label=task_dict['label'],
attributes=task_dict['attributes'],
--- a/taskcluster/taskgraph/task/nightly_fennec.py
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@@ -104,10 +104,10 @@ class NightlyFennecTask(base.Task):
# add a dependency on an image task, if needed
if 'docker-image' in self.task_dict:
deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
'docker-image'))
return deps
- def optimize(self):
+ def optimize(self, params):
return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@@ -46,10 +46,10 @@ class SigningTask(base.Task):
tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
attributes=attributes))
return tasks
def get_dependencies(self, taskgraph):
return [('build-nightly-fennec', 'build-nightly-fennec')]
- def optimize(self):
+ def optimize(self, params):
return False, None
--- a/taskcluster/taskgraph/task/transform.py
+++ b/taskcluster/taskgraph/task/transform.py
@@ -2,29 +2,30 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
from . import base
+from .. import files_changed
from ..util.python_path import find_object
from ..util.yaml import load_yaml
from ..transforms.base import TransformSequence, TransformConfig
logger = logging.getLogger(__name__)
class TransformTask(base.Task):
"""
Tasks of this class are generated by applying transformations to a sequence
of input entities. By default, it gets those inputs from YAML data in the
- kind directory, but subclasses may override `get_inputs` to produce them
- in some other way.
+ kind directory, but subclasses may override `get_inputs` to produce them in
+ some other way.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
"""
Get the input elements that will be transformed into tasks. The
elements themselves are free-form, and become the input to the first
transform.
@@ -58,16 +59,24 @@ class TransformTask(base.Task):
# perform the transformations
trans_config = TransformConfig(kind, path, config, params)
tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
return tasks
def __init__(self, kind, task):
self.dependencies = task['dependencies']
+ self.when = task['when']
super(TransformTask, self).__init__(kind, task['label'],
task['attributes'], task['task'])
def get_dependencies(self, taskgraph):
return [(label, name) for name, label in self.dependencies.items()]
- def optimize(self):
+ def optimize(self, params):
+ if 'files-changed' in self.when:
+ changed = files_changed.check(
+ params, self.when['files-changed'])
+ if not changed:
+ logger.debug('no files found matching a pattern in `when.files-changed` for '
+ + self.label)
+ return True, None
return False, None
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/automationrelevance.json
@@ -0,0 +1,425 @@
+{
+ "changesets": [
+ {
+ "author": "James Long <longster@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1300866",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
+ }
+ ],
+ "date": [
+ 1473196655.0,
+ 14400
+ ],
+ "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/debugger/new/index.html"
+ ],
+ "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "parents": [
+ "37c9349b4e8167a61b08b7e119c21ea177b98942"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312890,
+ "reviewers": [
+ {
+ "name": "jlast",
+ "revset": "reviewer(jlast)"
+ },
+ {
+ "name": "bgrins",
+ "revset": "reviewer(bgrins)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Wes Kocher <wkocher@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [],
+ "date": [
+ 1473208638.0,
+ 25200
+ ],
+ "desc": "Merge m-c to fx-team, a=merge",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "taskcluster/scripts/builder/build-l10n.sh"
+ ],
+ "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "parents": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312891,
+ "reviewers": [
+ {
+ "name": "merge",
+ "revset": "reviewer(merge)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Towkir Ahmed <towkir17@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1296648",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
+ }
+ ],
+ "date": [
+ 1472957580.0,
+ 14400
+ ],
+ "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/themes/rules.css"
+ ],
+ "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "parents": [
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312892,
+ "reviewers": [
+ {
+ "name": "ntim",
+ "revset": "reviewer(ntim)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Oriol <oriol-bugzilla@hotmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1300336",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
+ }
+ ],
+ "date": [
+ 1472921160.0,
+ 14400
+ ],
+ "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+ "devtools/server/actors/object.js"
+ ],
+ "node": "99c542fa43a72ee863c813b5624048d1b443549b",
+ "parents": [
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312893,
+ "reviewers": [
+ {
+ "name": "fitzgen",
+ "revset": "reviewer(fitzgen)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Ruturaj Vartak <ruturaj@gmail.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1295010",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+ }
+ ],
+ "date": [
+ 1472854020.0,
+ -7200
+ ],
+ "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
+ "extra": {
+ "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
+ "branch": "default",
+ "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
+ "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
+ },
+ "files": [
+ "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+ "devtools/client/inspector/test/head.js",
+ "devtools/server/actors/highlighters/eye-dropper.js"
+ ],
+ "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "parents": [
+ "99c542fa43a72ee863c813b5624048d1b443549b"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312894,
+ "reviewers": [
+ {
+ "name": "pbro",
+ "revset": "reviewer(pbro)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Matteo Ferretti <mferretti@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1299154",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
+ }
+ ],
+ "date": [
+ 1472629906.0,
+ -7200
+ ],
+ "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
+ "extra": {
+ "branch": "default",
+ "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
+ },
+ "files": [
+ "docshell/base/nsDocShell.cpp",
+ "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
+ ],
+ "node": "541c9086c0f27fba60beecc9bc94543103895c86",
+ "parents": [
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312895,
+ "reviewers": [
+ {
+ "name": "mstange",
+ "revset": "reviewer(mstange)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Patrick Brosset <pbrosset@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [
+ {
+ "no": "1295010",
+ "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+ }
+ ],
+ "date": [
+ 1473239449.0,
+ -7200
+ ],
+ "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [
+ "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+ "devtools/client/inspector/test/head.js"
+ ],
+ "node": "041a925171e431bf51fb50193ab19d156088c89a",
+ "parents": [
+ "541c9086c0f27fba60beecc9bc94543103895c86"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312896,
+ "reviewers": [
+ {
+ "name": "me",
+ "revset": "reviewer(me)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ },
+ {
+ "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
+ "backsoutnodes": [],
+ "bugs": [],
+ "date": [
+ 1473261233.0,
+ -7200
+ ],
+ "desc": "merge fx-team to mozilla-central a=merge",
+ "extra": {
+ "branch": "default"
+ },
+ "files": [],
+ "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "parents": [
+ "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
+ "041a925171e431bf51fb50193ab19d156088c89a"
+ ],
+ "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "pushdate": [
+ 1473261248,
+ 0
+ ],
+ "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+ "pushid": 30664,
+ "pushnodes": [
+ "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+ "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+ "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+ "99c542fa43a72ee863c813b5624048d1b443549b",
+ "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+ "541c9086c0f27fba60beecc9bc94543103895c86",
+ "041a925171e431bf51fb50193ab19d156088c89a",
+ "a14f88a9af7a59e677478694bafd9375ac53683e"
+ ],
+ "pushuser": "cbook@mozilla.com",
+ "rev": 312897,
+ "reviewers": [
+ {
+ "name": "merge",
+ "revset": "reviewer(merge)"
+ }
+ ],
+ "treeherderrepo": "mozilla-central",
+ "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+ }
+ ],
+ "visible": true
+}
+
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_files_changed.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import unittest
+
+from .. import files_changed
+
+PARAMS = {
+ 'head_repository': 'https://hg.mozilla.org/mozilla-central',
+ 'head_rev': 'a14f88a9af7a',
+}
+
+FILES_CHANGED = [
+ 'devtools/client/debugger/new/index.html',
+ 'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js',
+ 'devtools/client/inspector/test/head.js',
+ 'devtools/client/themes/rules.css',
+ 'devtools/client/webconsole/test/browser_webconsole_output_06.js',
+ 'devtools/server/actors/highlighters/eye-dropper.js',
+ 'devtools/server/actors/object.js',
+ 'docshell/base/nsDocShell.cpp',
+ 'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html',
+ 'taskcluster/scripts/builder/build-l10n.sh',
+]
+
+
+class FakeResponse:
+
+ def json(self):
+ with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f:
+ return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+
+ def setUp(self):
+ files_changed._cache.clear()
+ self.old_get = files_changed.requests.get
+
+ def fake_get(url, **kwargs):
+ return FakeResponse()
+ files_changed.requests.get = fake_get
+
+ def tearDown(self):
+ files_changed.requests.get = self.old_get
+
+ def test_get_changed_files(self):
+ """Get_changed_files correctly gets the list of changed files in a push.
+ This tests against the production hg.mozilla.org so that it will detect
+ any changes in the format of the returned data."""
+ self.assertEqual(
+ sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])),
+ FILES_CHANGED)
+
+
+class TestCheck(unittest.TestCase):
+
+ def setUp(self):
+ files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED
+
+ def test_check_no_params(self):
+ self.assertTrue(files_changed.check({}, ["ignored"]))
+
+ def test_check_no_match(self):
+ self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+
+ def test_check_match(self):
+ self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -29,17 +29,17 @@ class FakeTask(base.Task):
def get_dependencies(self, full_task_set):
i = self.i
if i > 0:
return [('{}-t-{}'.format(self.kind, i - 1), 'prev')]
else:
return []
- def optimize(self):
+ def optimize(self, params):
return False, None
class FakeKind(Kind):
def _get_impl_class(self):
return FakeTask
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -80,85 +80,85 @@ class TestOptimize(unittest.TestCase):
return 'SLUGID' if task_id and len(task_id) == 22 else task_id
got_annotations = {
t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
}
self.assertEqual(got_annotations, annotations)
def test_annotate_task_graph_no_optimize(self):
"annotating marks everything as un-optimized if the kind returns that"
- OptimizingTask.optimize = lambda self: (False, None)
+ OptimizingTask.optimize = lambda self, params: (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
self.make_task('task3'),
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
- annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+ annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
self.assert_annotations(
graph,
task1=(False, None),
task2=(False, None),
task3=(False, None)
)
def test_annotate_task_graph_taskid_without_optimize(self):
"raises exception if kind returns a taskid without optimizing"
- OptimizingTask.optimize = lambda self: (False, 'some-taskid')
+ OptimizingTask.optimize = lambda self, params: (False, 'some-taskid')
graph = self.make_graph(self.make_task('task1'))
self.assertRaises(
Exception,
- lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+ lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
)
def test_annotate_task_graph_optimize_away_dependency(self):
"raises exception if kind optimizes away a task on which another depends"
OptimizingTask.optimize = \
- lambda self: (True, None) if self.label == 'task1' else (False, None)
+ lambda self, params: (True, None) if self.label == 'task1' else (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
('task2', 'task1', 'build'),
)
self.assertRaises(
Exception,
- lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
+ lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
)
def test_annotate_task_graph_do_not_optimize(self):
"annotating marks everything as un-optimized if in do_not_optimize"
- OptimizingTask.optimize = lambda self: (True, 'taskid')
+ OptimizingTask.optimize = lambda self, params: (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
('task2', 'task1', 'build'),
)
label_to_taskid = {}
- annotate_task_graph(graph, {'task1', 'task2'},
+ annotate_task_graph(graph, {}, {'task1', 'task2'},
graph.graph.named_links_dict(), label_to_taskid, None)
self.assert_annotations(
graph,
task1=(False, None),
task2=(False, None)
)
self.assertEqual
def test_annotate_task_graph_nos_do_not_propagate(self):
"a task with a non-optimized dependency can be optimized"
OptimizingTask.optimize = \
- lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
+ lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
self.make_task('task3'),
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
- annotate_task_graph(graph, set(),
+ annotate_task_graph(graph, {}, set(),
graph.graph.named_links_dict(), {}, None)
self.assert_annotations(
graph,
task1=(False, None),
task2=(True, 'taskid'),
task3=(True, 'taskid')
)
@@ -237,20 +237,20 @@ class TestOptimize(unittest.TestCase):
self.assertEqual(sub.tasks[task2].task_id, task2)
self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1']))
self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3)
self.assertEqual(sub.tasks[task3].task_id, task3)
def test_optimize(self):
"optimize_task_graph annotates and extracts the subgraph from a simple graph"
OptimizingTask.optimize = \
- lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
+ lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None)
input = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
self.make_task('task3'),
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
- opt, label_to_taskid = optimize_task_graph(input, set())
+ opt, label_to_taskid = optimize_task_graph(input, {}, set())
self.assertEqual(opt.graph, graph.Graph(
{label_to_taskid['task2'], label_to_taskid['task3']},
{(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -49,16 +49,17 @@ job_description_schema = Schema({
Optional('scopes'): task_description_schema['scopes'],
Optional('extra'): task_description_schema['extra'],
Optional('treeherder'): task_description_schema['treeherder'],
Optional('index'): task_description_schema['index'],
Optional('run-on-projects'): task_description_schema['run-on-projects'],
Optional('coalesce-name'): task_description_schema['coalesce-name'],
Optional('worker-type'): task_description_schema['worker-type'],
Required('worker'): task_description_schema['worker'],
+ Optional('when'): task_description_schema['when'],
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -207,16 +207,26 @@ task_description_schema = Schema({
Optional('repository'): basestring,
Optional('project'): basestring,
},
'properties': {
'product': basestring,
Extra: basestring, # additional properties are allowed
},
}),
+
+ # The "when" section contains descriptions of the circumstances
+ # under which this task can be "optimized", that is, left out of the
+ # task graph because it is unnecessary.
+ Optional('when'): Any({
+ # This task only needs to be run if a file matching one of the given
+ # patterns has changed in the push. The patterns use the mozpack
+ # match function (python/mozbuild/mozpack/path.py).
+ Optional('files-changed'): [basestring],
+ }),
})
GROUP_NAMES = {
'tc': 'Executed by TaskCluster',
'tc-e10s': 'Executed by TaskCluster with e10s',
'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',