Bug 1249091 - Add IMPACTED_TASKS to moz.build Files metadata; r?glandium
Bug 1245953 added support for limiting TaskCluster task scheduling to
when certain files change. This was implemented as a list of mozpath
patterns in TaskCluster's YAML files. A drawback of that approach is
YAML files in testing/taskcluster list paths from all over the tree.
This is prone to getting out of sync and can be a pain to update.
moz.build files already provide a mechanism for associating metadata
with files. And TaskCluster's scheduling code runs on a full source code
checkout and it knows which files have changed. Putting all that
together means it is possible for TaskCluster to query moz.build files
for the set of tasks that are relevant to a file.
This commit introduces the IMPACTED_TASKS Files metadata variable for
declaring which TaskCluster tasks are impacted by files [and should be
scheduled]. A subsequent commit will teach the TaskCluster code to query
this metadata.
MozReview-Commit-ID: IOSu9kMcSvi
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -651,16 +651,17 @@ class Files(SubContext):
further updates to a variable.
When ``FINAL`` is set, the value of all variables defined in this
context are marked as frozen and all subsequent writes to them
are ignored during metadata reading.
See :ref:`mozbuild_files_metadata_finalizing` for more info.
"""),
+
'IMPACTED_TESTS': (DependentTestsEntry, list,
"""File patterns, tags, and flavors for tests relevant to these files.
Maps source files to the tests potentially impacted by those files.
Tests can be specified by file pattern, tag, or flavor.
For example:
@@ -699,16 +700,26 @@ class Files(SubContext):
with Files('dom/base/nsGlobalWindow.cpp'):
IMPACTED_TESTS.flavors += [
'mochitest',
]
Would suggest that nsGlobalWindow.cpp is potentially relevant to
any plain mochitest.
"""),
+
+ 'IMPACTED_TASKS': (StrictOrderingOnAppendList, list,
+ """TaskCluster tasks that should be executed when files change.
+
+ Not all automation tasks need to run all the time. Many automation
+ tasks only need to run after certain files change.
+
+ This variable stores a list of Task Cluster task names that are
+ impacted and should be executed when certain files change.
+ """),
}
def __init__(self, parent, pattern=None):
super(Files, self).__init__(parent)
self.pattern = pattern
self.finalized = set()
self.test_files = set()
self.test_tags = set()
@@ -733,16 +744,20 @@ class Files(SubContext):
if k in self.finalized:
continue
# Only finalize variables defined in this instance.
if k == 'FINAL':
self.finalized |= set(other) - {'FINAL'}
continue
+ if k == 'IMPACTED_TASKS':
+ self[k] += sorted(v)
+ continue
+
self[k] = v
return self
def asdict(self):
"""Return this instance as a dict with built-in data structures.
Call this to obtain an object suitable for serializing.
@@ -769,37 +784,42 @@ class Files(SubContext):
:py:func:`mozbuild.frontend.context.Files` instances passed in are
thus the "collapsed" (``__iadd__``ed) results of all ``Files`` from all
moz.build files relevant to a specific path, not individual ``Files``
instances from a single moz.build file.
"""
d = {}
bug_components = Counter()
+ impacted_tasks = set()
for f in files.values():
bug_component = f.get('BUG_COMPONENT')
if bug_component:
bug_components[bug_component] += 1
+ impacted_tasks |= set(f.get('IMPACTED_TASKS', []))
+
d['bug_component_counts'] = []
for c, count in bug_components.most_common():
component = (c.product, c.component)
d['bug_component_counts'].append((c, count))
if 'recommended_bug_component' not in d:
d['recommended_bug_component'] = component
recommended_count = count
elif count == recommended_count:
# Don't recommend a component if it doesn't have a clear lead.
d['recommended_bug_component'] = None
# In case no bug components.
d.setdefault('recommended_bug_component', None)
+ d['impacted_tasks'] = sorted(impacted_tasks)
+
return d
# This defines functions that create sub-contexts.
#
# Values are classes that are SubContexts. The class name will be turned into
# a function that when called emits an instance of that class.
#
--- a/python/mozbuild/mozbuild/frontend/mach_commands.py
+++ b/python/mozbuild/mozbuild/frontend/mach_commands.py
@@ -153,16 +153,34 @@ class MozbuildFileCommands(MachCommandBa
print('\tRelevant flavors:')
for p in m.test_flavors:
print('\t\t%s' % p)
except InvalidPathException as e:
print(e.message)
return 1
+ @SubCommand('file-info', 'impacted-tasks',
+ 'Show TaskCluster tasks impacted by changes to files')
+ @CommandArgument('-r', '--rev',
+ help='Version control revision to look up info from')
+ @CommandArgument('paths', nargs='*',
+ help='Paths whose data to query')
+ def file_info_impacted_tasks(self, paths, rev=None):
+ try:
+ for p, m in sorted(self._get_files_info(paths, rev=rev).items()):
+ tasks = m.get('IMPACTED_TASKS', [])
+ if not tasks:
+ continue
+
+ relpath = mozpath.relpath(p, self.topsrcdir)
+ print('%s: %s' % (relpath, ', '.join(sorted(tasks))))
+ except InvalidPathException as e:
+ print(e.message)
+ return 1
def _get_reader(self, finder):
from mozbuild.frontend.reader import (
BuildReader,
EmptyConfig,
)
config = EmptyConfig(self.topsrcdir)
--- a/python/mozbuild/mozbuild/test/frontend/test_context.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_context.py
@@ -662,27 +662,29 @@ class TestTypedRecord(unittest.TestCase)
class TestFiles(unittest.TestCase):
def test_aggregate_empty(self):
c = Context({})
files = {'moz.build': Files(c, pattern='**')}
self.assertEqual(Files.aggregate(files), {
'bug_component_counts': [],
+ 'impacted_tasks': [],
'recommended_bug_component': None,
})
def test_single_bug_component(self):
c = Context({})
f = Files(c, pattern='**')
f['BUG_COMPONENT'] = (u'Product1', u'Component1')
files = {'moz.build': f}
self.assertEqual(Files.aggregate(files), {
'bug_component_counts': [((u'Product1', u'Component1'), 1)],
+ 'impacted_tasks': [],
'recommended_bug_component': (u'Product1', u'Component1'),
})
def test_multiple_bug_components(self):
c = Context({})
f1 = Files(c, pattern='**')
f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
@@ -690,16 +692,17 @@ class TestFiles(unittest.TestCase):
f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
files = {'a': f1, 'b': f2, 'c': f1}
self.assertEqual(Files.aggregate(files), {
'bug_component_counts': [
((u'Product1', u'Component1'), 2),
((u'Product2', u'Component2'), 1),
],
+ 'impacted_tasks': [],
'recommended_bug_component': (u'Product1', u'Component1'),
})
def test_no_recommended_bug_component(self):
"""If there is no clear count winner, we don't recommend a bug component."""
c = Context({})
f1 = Files(c, pattern='**')
f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
@@ -708,14 +711,28 @@ class TestFiles(unittest.TestCase):
f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
files = {'a': f1, 'b': f2}
self.assertEqual(Files.aggregate(files), {
'bug_component_counts': [
((u'Product1', u'Component1'), 1),
((u'Product2', u'Component2'), 1),
],
+ 'impacted_tasks': [],
'recommended_bug_component': None,
})
+ def test_impacted_tasks_aggregate(self):
+ c = Context({})
+ f1 = Files(c, pattern='**')
+ f1['IMPACTED_TASKS'] += ['task1', 'task2']
+ f2 = Files(c, pattern='**')
+ f2['IMPACTED_TASKS'] += ['task0', 'task3']
+
+ files = {'a': f1, 'b': f2}
+ self.assertEqual(Files.aggregate(files), {
+ 'bug_component_counts': [],
+ 'impacted_tasks': ['task0', 'task1', 'task2', 'task3'],
+ 'recommended_bug_component': None,
+ })
if __name__ == '__main__':
main()