Bug 1402010 - Support not reading test manifests in moz.build files; r?chmanchester draft
authorGregory Szorc <gps@mozilla.com>
Thu, 21 Sep 2017 11:40:08 -0700
changeset 668490 8d29a05600bfadc5f592bd6973efdd3d4124af36
parent 668487 23dbfc6d8ebf99c5c3c49656acd9ba644a6a2637
child 732720 448cb69429171e277ad0e810155fff9d75408147
push id81064
push usergszorc@mozilla.com
push dateThu, 21 Sep 2017 18:41:04 +0000
reviewerschmanchester
bugs1402010
milestone58.0a1
Bug 1402010 - Support not reading test manifests in moz.build files; r?chmanchester Not all consumers of moz.build evaluation are interested in test manifest data. Normally, processing it isn't a big deal so we just do it. However, there are a few scenarios where this is difficult. The difficulty processing test manifests stems from the fact that test manifest parsers aren't part of the mozbuild Python package. The Python modules are defined elsewhere in the repo. In the case of reading moz.build files from version control, a full checkout may not be available or the calling process may not be running from the specific revision being evaluated. In either case, these manifest processing modules may not be available and moz.build evaluation will fail. This commit introduces a flag on BuildReader.files_info() to control whether test manifests are loaded. If disabled, an evaluation flag is added and when the respective moz.build variables are evaluated, they see the flag and short-circuit. The module imports in testing.py were also refactored as part of this change to swallow any import failure. We had previously delay imported these modules as a way to work around the import failure in certain context. If the import fails, an exception will be thrown attempting to operate on None. This is slightly more annoying than an explicit ImportError. The reason we can't keep the import in the function is because "import" is processed at block scoping, so if an import is in a block, it gets processed, even if that line is never evaluated. Various callers not needing to access test manifest data have been changed to not load it. A side effect of this change is that various `mach file-info` commands became faster! For example, `mach file-info bugzilla-component 'testing/**'` dropped ~500ms from ~9000ms. MozReview-Commit-ID: 9mtBg4AWCc
python/mozbuild/mozbuild/frontend/mach_commands.py
python/mozbuild/mozbuild/frontend/reader.py
python/mozbuild/mozbuild/testing.py
taskcluster/taskgraph/optimize.py
--- a/python/mozbuild/mozbuild/frontend/mach_commands.py
+++ b/python/mozbuild/mozbuild/frontend/mach_commands.py
@@ -101,17 +101,19 @@ class MozbuildFileCommands(MachCommandBa
     def file_info_bugzilla(self, paths, rev=None):
         """Show Bugzilla component for a set of files.
 
         Given a requested set of files (which can be specified using
         wildcards), print the Bugzilla component for each file.
         """
         components = defaultdict(set)
         try:
-            for p, m in self._get_files_info(paths, rev=rev).items():
+            res = self._get_files_info(paths, rev=rev,
+                                       read_test_manifests=False)
+            for p, m in res.items():
                 components[m.get('BUG_COMPONENT')].add(p)
         except InvalidPathException as e:
             print(e.message)
             return 1
 
         for component, files in sorted(components.items(), key=lambda x: (x is None, x)):
             print('%s :: %s' % (component.product, component.component) if component else 'UNKNOWN')
             for f in sorted(files):
@@ -120,17 +122,19 @@ class MozbuildFileCommands(MachCommandBa
     @SubCommand('file-info', 'missing-bugzilla',
                 'Show files missing Bugzilla component info')
     @CommandArgument('-r', '--rev',
                      help='Version control revision to look up info from')
     @CommandArgument('paths', nargs='+',
                      help='Paths whose data to query')
     def file_info_missing_bugzilla(self, paths, rev=None):
         try:
-            for p, m in sorted(self._get_files_info(paths, rev=rev).items()):
+            res = self._get_files_info(paths, rev=rev,
+                                       read_test_manifests=False)
+            for p, m in sorted(res.items()):
                 if 'BUG_COMPONENT' not in m:
                     print(p)
         except InvalidPathException as e:
             print(e.message)
             return 1
 
     @SubCommand('file-info', 'dep-tests',
                 'Show test files marked as dependencies of these source files.')
@@ -155,17 +159,17 @@ class MozbuildFileCommands(MachCommandBa
                     for p in m.test_flavors:
                         print('\t\t%s' % p)
 
         except InvalidPathException as e:
             print(e.message)
             return 1
 
 
-    def _get_files_info(self, paths, rev=None):
+    def _get_files_info(self, paths, rev=None, read_test_manifests=True):
         reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev)
 
         # Normalize to relative from topsrcdir.
         query_paths = []
         for p in paths:
             a = mozpath.abspath(p)
             if not mozpath.basedir(a, [self.topsrcdir]):
                 raise InvalidPathException('path is outside topsrcdir: %s' % p)
@@ -187,29 +191,30 @@ class MozbuildFileCommands(MachCommandBa
             if rev:
                 raise InvalidPathException('cannot use wildcard in version control mode')
 
             for path, f in reader.finder.find(p):
                 if path not in all_paths_set:
                     all_paths_set.add(path)
                     allpaths.append(path)
 
-        return reader.files_info(allpaths)
+        return reader.files_info(allpaths,
+                                 read_test_manifests=read_test_manifests)
 
 
     @SubCommand('file-info', 'schedules',
                 'Show the combined SCHEDULES for the files listed.')
     @CommandArgument('paths', nargs='+',
                      help='Paths whose data to query')
     def file_info_schedules(self, paths):
         """Show what is scheduled by the given files.
 
         Given a requested set of files (which can be specified using
         wildcards), print the total set of scheduled components.
         """
         from mozbuild.frontend.reader import EmptyConfig, BuildReader
         config = EmptyConfig(TOPSRCDIR)
         reader = BuildReader(config)
         schedules = set()
-        for p, m in reader.files_info(paths).items():
+        for p, m in reader.files_info(paths, read_test_manifests=False).items():
             schedules |= set(m['SCHEDULES'].components)
 
         print(", ".join(schedules))
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1329,36 +1329,43 @@ class BuildReader(object):
             all_contexts.append(context)
 
         result = {}
         for path, paths in path_mozbuilds.items():
             result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), [])
 
         return result, all_contexts
 
-    def files_info(self, paths):
+    def files_info(self, paths, read_test_manifests=True):
         """Obtain aggregate data from Files for a set of files.
 
         Given a set of input paths, determine which moz.build files may
         define metadata for them, evaluate those moz.build files, and
         apply file metadata rules defined within to determine metadata
         values for each file requested.
 
         Essentially, for each input path:
 
         1. Determine the set of moz.build files relevant to that file by
            looking for moz.build files in ancestor directories.
         2. Evaluate moz.build files starting with the most distant.
         3. Iterate over Files sub-contexts.
         4. If the file pattern matches the file we're seeking info on,
            apply attribute updates.
         5. Return the most recent value of attributes.
+
+        ``read_test_manifests`` can be set to false to disable the loading
+        of test manifest files. This can speed up moz.build evaluation and
+        limit the number of required imports.
         """
         eval_flags = {'files-info'}
 
+        if not read_test_manifests:
+            eval_flags.add('no-test-manifests')
+
         paths, _ = self.read_relevant_mozbuilds(paths, eval_flags=eval_flags)
 
         # For thousands of inputs (say every file in a sub-tree),
         # test_defaults_for_path() gets called with the same contexts multiple
         # times (once for every path in a directory that doesn't have any
         # test metadata). So, we cache the function call.
         defaults_cache = {}
         def test_defaults_for_path(ctxs):
@@ -1397,18 +1404,20 @@ class BuildReader(object):
 
                 # Only do wildcard matching if the '*' character is present.
                 # Otherwise, mozpath.match will match directories, which we've
                 # arbitrarily chosen to not allow.
                 if pattern == relpath or \
                         ('*' in pattern and mozpath.match(relpath, pattern)):
                     flags += ctx
 
-            if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
-                flags += test_defaults_for_path(ctxs)
+            if read_test_manifests:
+                if not any([flags.test_tags, flags.test_files,
+                            flags.test_flavors]):
+                    flags += test_defaults_for_path(ctxs)
 
             r[path] = flags
 
         return r
 
     def test_defaults_for_path(self, ctxs):
         # This names the context keys that will end up emitting a test
         # manifest.
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -12,17 +12,26 @@ import mozpack.path as mozpath
 
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
 
 from .base import MozbuildObject
 from .util import OrderedDefaultDict
 from collections import defaultdict
 
-import manifestparser
+try:
+    import manifestparser
+except ImportError:
+    manifestparser = None
+
+try:
+    import reftest
+except ImportError:
+    reftest = None
+
 
 def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
     """Rewrite paths in a test to be under a new base path.
 
     This is useful for running tests from a separate location from where they
     were defined.
 
     honor_install_to_subdir and the underlying install-to-subdir field are a
@@ -504,30 +513,39 @@ def install_test_files(topsrcdir, topobj
     copier = FileCopier()
     manifest.populate_registry(copier)
     copier.copy(objdir_dest,
                 remove_unaccounted=False)
 
 
 # Convenience methods for test manifest reading.
 def read_manifestparser_manifest(context, manifest_path):
+    if 'no-test-manifests' in context.eval_flags:
+        return
+
     path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
     return manifestparser.TestManifest(manifests=[path], strict=True,
                                        rootdir=context.config.topsrcdir,
                                        finder=context._finder,
                                        handle_defaults=False)
 
 def read_reftest_manifest(context, manifest_path):
-    import reftest
+    if 'no-test-manifests' in context.eval_flags:
+        return
+
     path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
     manifest = reftest.ReftestManifest(finder=context._finder)
     manifest.load(path)
     return manifest
 
+
 def read_wpt_manifest(context, paths):
+    if 'no-test-manifests' in context.eval_flags:
+        return
+
     manifest_path, tests_root = paths
     full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
     old_path = sys.path[:]
     try:
         # Setup sys.path to include all the dependencies required to import
         # the web-platform-tests manifest parser. web-platform-tests provides
         # a the localpaths.py to do the path manipulation, which we load,
         # providing the __file__ variable so it can resolve the relative
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -359,17 +359,18 @@ class SkipUnlessSchedules(OptimizationSt
 
         mbo = MozbuildObject.from_environment()
         # the decision task has a sparse checkout, so, mozbuild_reader will use
         # a MercurialRevisionFinder with revision '.', which should be the same
         # as `revision`; in other circumstances, it will use a default reader
         rdr = mbo.mozbuild_reader(config_mode='empty')
 
         components = set()
-        for p, m in rdr.files_info(changed_files).items():
+        res = rdr.files_info(changed_files, read_test_manifests=False)
+        for p, m in res.items():
             components |= set(m['SCHEDULES'].components)
 
         return components
 
     def should_remove_task(self, task, params, conditions):
         if params.get('pushlog_id') == -1:
             return False