Bug 1402010 - Support not reading test manifests in moz.build files; r?chmanchester
Not all consumers of moz.build evaluation are interested in test
manifest data. Normally, processing it isn't a big deal so we
just do it. However, there are a few scenarios where this is
difficult.
The difficulty processing test manifests stems from the fact
that test manifest parsers aren't part of the mozbuild Python
package. The Python modules are defined elsewhere in the repo.
In the case of reading moz.build files from version control,
a full checkout may not be available or the calling process may
not be running from the specific revision being evaluated. In
either case, these manifest processing modules may not be
available and moz.build evaluation will fail.
This commit introduces a flag on BuildReader.files_info() to
control whether test manifests are loaded. If disabled, an
evaluation flag is added and when the respective moz.build
variables are evaluated, they see the flag and short-circuit.
The module imports in testing.py were also refactored as part
of this change to swallow any import failure. We had previously
delay imported these modules as a way to work around the import
failure in certain context. If the import fails, an exception
will be thrown attempting to operate on None. This is slightly
more annoying than an explicit ImportError. The reason we can't
keep the import in the function is because "import" is processed
at block scoping, so if an import is in a block, it gets
processed, even if that line is never evaluated.
Various callers not needing to access test manifest data have
been changed to not load it.
A side effect of this change is that various `mach file-info`
commands became faster! For example,
`mach file-info bugzilla-component 'testing/**'` dropped ~500ms
from ~9000ms.
MozReview-Commit-ID: 9mtBg4AWCc
--- a/python/mozbuild/mozbuild/frontend/mach_commands.py
+++ b/python/mozbuild/mozbuild/frontend/mach_commands.py
@@ -101,17 +101,19 @@ class MozbuildFileCommands(MachCommandBa
def file_info_bugzilla(self, paths, rev=None):
"""Show Bugzilla component for a set of files.
Given a requested set of files (which can be specified using
wildcards), print the Bugzilla component for each file.
"""
components = defaultdict(set)
try:
- for p, m in self._get_files_info(paths, rev=rev).items():
+ res = self._get_files_info(paths, rev=rev,
+ read_test_manifests=False)
+ for p, m in res.items():
components[m.get('BUG_COMPONENT')].add(p)
except InvalidPathException as e:
print(e.message)
return 1
for component, files in sorted(components.items(), key=lambda x: (x is None, x)):
print('%s :: %s' % (component.product, component.component) if component else 'UNKNOWN')
for f in sorted(files):
@@ -120,17 +122,19 @@ class MozbuildFileCommands(MachCommandBa
@SubCommand('file-info', 'missing-bugzilla',
'Show files missing Bugzilla component info')
@CommandArgument('-r', '--rev',
help='Version control revision to look up info from')
@CommandArgument('paths', nargs='+',
help='Paths whose data to query')
def file_info_missing_bugzilla(self, paths, rev=None):
try:
- for p, m in sorted(self._get_files_info(paths, rev=rev).items()):
+ res = self._get_files_info(paths, rev=rev,
+ read_test_manifests=False)
+ for p, m in sorted(res.items()):
if 'BUG_COMPONENT' not in m:
print(p)
except InvalidPathException as e:
print(e.message)
return 1
@SubCommand('file-info', 'dep-tests',
'Show test files marked as dependencies of these source files.')
@@ -155,17 +159,17 @@ class MozbuildFileCommands(MachCommandBa
for p in m.test_flavors:
print('\t\t%s' % p)
except InvalidPathException as e:
print(e.message)
return 1
- def _get_files_info(self, paths, rev=None):
+ def _get_files_info(self, paths, rev=None, read_test_manifests=True):
reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev)
# Normalize to relative from topsrcdir.
query_paths = []
for p in paths:
a = mozpath.abspath(p)
if not mozpath.basedir(a, [self.topsrcdir]):
raise InvalidPathException('path is outside topsrcdir: %s' % p)
@@ -187,29 +191,30 @@ class MozbuildFileCommands(MachCommandBa
if rev:
raise InvalidPathException('cannot use wildcard in version control mode')
for path, f in reader.finder.find(p):
if path not in all_paths_set:
all_paths_set.add(path)
allpaths.append(path)
- return reader.files_info(allpaths)
+ return reader.files_info(allpaths,
+ read_test_manifests=read_test_manifests)
@SubCommand('file-info', 'schedules',
'Show the combined SCHEDULES for the files listed.')
@CommandArgument('paths', nargs='+',
help='Paths whose data to query')
def file_info_schedules(self, paths):
"""Show what is scheduled by the given files.
Given a requested set of files (which can be specified using
wildcards), print the total set of scheduled components.
"""
from mozbuild.frontend.reader import EmptyConfig, BuildReader
config = EmptyConfig(TOPSRCDIR)
reader = BuildReader(config)
schedules = set()
- for p, m in reader.files_info(paths).items():
+ for p, m in reader.files_info(paths, read_test_manifests=False).items():
schedules |= set(m['SCHEDULES'].components)
print(", ".join(schedules))
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1329,36 +1329,43 @@ class BuildReader(object):
all_contexts.append(context)
result = {}
for path, paths in path_mozbuilds.items():
result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), [])
return result, all_contexts
- def files_info(self, paths):
+ def files_info(self, paths, read_test_manifests=True):
"""Obtain aggregate data from Files for a set of files.
Given a set of input paths, determine which moz.build files may
define metadata for them, evaluate those moz.build files, and
apply file metadata rules defined within to determine metadata
values for each file requested.
Essentially, for each input path:
1. Determine the set of moz.build files relevant to that file by
looking for moz.build files in ancestor directories.
2. Evaluate moz.build files starting with the most distant.
3. Iterate over Files sub-contexts.
4. If the file pattern matches the file we're seeking info on,
apply attribute updates.
5. Return the most recent value of attributes.
+
+ ``read_test_manifests`` can be set to false to disable the loading
+ of test manifest files. This can speed up moz.build evaluation and
+ limit the number of required imports.
"""
eval_flags = {'files-info'}
+ if not read_test_manifests:
+ eval_flags.add('no-test-manifests')
+
paths, _ = self.read_relevant_mozbuilds(paths, eval_flags=eval_flags)
# For thousands of inputs (say every file in a sub-tree),
# test_defaults_for_path() gets called with the same contexts multiple
# times (once for every path in a directory that doesn't have any
# test metadata). So, we cache the function call.
defaults_cache = {}
def test_defaults_for_path(ctxs):
@@ -1397,18 +1404,20 @@ class BuildReader(object):
# Only do wildcard matching if the '*' character is present.
# Otherwise, mozpath.match will match directories, which we've
# arbitrarily chosen to not allow.
if pattern == relpath or \
('*' in pattern and mozpath.match(relpath, pattern)):
flags += ctx
- if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
- flags += test_defaults_for_path(ctxs)
+ if read_test_manifests:
+ if not any([flags.test_tags, flags.test_files,
+ flags.test_flavors]):
+ flags += test_defaults_for_path(ctxs)
r[path] = flags
return r
def test_defaults_for_path(self, ctxs):
# This names the context keys that will end up emitting a test
# manifest.
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -12,17 +12,26 @@ import mozpack.path as mozpath
from mozpack.copier import FileCopier
from mozpack.manifests import InstallManifest
from .base import MozbuildObject
from .util import OrderedDefaultDict
from collections import defaultdict
-import manifestparser
+try:
+ import manifestparser
+except ImportError:
+ manifestparser = None
+
+try:
+ import reftest
+except ImportError:
+ reftest = None
+
def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
"""Rewrite paths in a test to be under a new base path.
This is useful for running tests from a separate location from where they
were defined.
honor_install_to_subdir and the underlying install-to-subdir field are a
@@ -504,30 +513,39 @@ def install_test_files(topsrcdir, topobj
copier = FileCopier()
manifest.populate_registry(copier)
copier.copy(objdir_dest,
remove_unaccounted=False)
# Convenience methods for test manifest reading.
def read_manifestparser_manifest(context, manifest_path):
+ if 'no-test-manifests' in context.eval_flags:
+ return
+
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
return manifestparser.TestManifest(manifests=[path], strict=True,
rootdir=context.config.topsrcdir,
finder=context._finder,
handle_defaults=False)
def read_reftest_manifest(context, manifest_path):
- import reftest
+ if 'no-test-manifests' in context.eval_flags:
+ return
+
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
manifest = reftest.ReftestManifest(finder=context._finder)
manifest.load(path)
return manifest
+
def read_wpt_manifest(context, paths):
+ if 'no-test-manifests' in context.eval_flags:
+ return
+
manifest_path, tests_root = paths
full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
old_path = sys.path[:]
try:
# Setup sys.path to include all the dependencies required to import
# the web-platform-tests manifest parser. web-platform-tests provides
# a the localpaths.py to do the path manipulation, which we load,
# providing the __file__ variable so it can resolve the relative
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -359,17 +359,18 @@ class SkipUnlessSchedules(OptimizationSt
mbo = MozbuildObject.from_environment()
# the decision task has a sparse checkout, so, mozbuild_reader will use
# a MercurialRevisionFinder with revision '.', which should be the same
# as `revision`; in other circumstances, it will use a default reader
rdr = mbo.mozbuild_reader(config_mode='empty')
components = set()
- for p, m in rdr.files_info(changed_files).items():
+ res = rdr.files_info(changed_files, read_test_manifests=False)
+ for p, m in res.items():
components |= set(m['SCHEDULES'].components)
return components
def should_remove_task(self, task, params, conditions):
if params.get('pushlog_id') == -1:
return False