Bug 1320194 - Generate all-tests.pkl and related files when resolving tests draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 01 Feb 2017 09:56:09 -0500
changeset 482039 25a2d85e93d36684df183ebf79b58ebc65ac2dd9
parent 482038 0edf877a6166e85b16216b65bba32dddda087d97
child 545345 958eae4e215da1996f18a51115225b2165bad063
push id44987
push userahalberstadt@mozilla.com
push dateFri, 10 Feb 2017 21:37:10 +0000
bugs1320194
milestone54.0a1
Bug 1320194 - Generate all-tests.pkl and related files when resolving tests This replaces the 'run-tests-deps' make target with a python function that will directly read moz.build files, emit them with TestManifestEmitter, then consume them with TestManifestBackend. Because the TestResolver is the only place that actually reads the test metadata files, we can remove this logic from the CommonBackend as well. MozReview-Commit-ID: DXgMoeH5dKf MozReview-Commit-ID: HstZ57qkqf2
Makefile.in
build/gen_test_backend.py
python/moz.build
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/testing.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -181,21 +181,16 @@ endif
 	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
 .PHONY: install-tests
 install-tests: install-test-files
 
-# We no longer run "make install-tests" directly before running tests, but we still
-# want to depend on things like config.status, hence this target.
-.PHONY: run-tests-deps
-run-tests-deps: $(install_manifest_depends)
-
 # Force --no-remove, because $objdir/_tests is handled by multiple manifests.
 .PHONY: install-test-files
 install-test-files:
 	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
 # dist and _tests should be purged during cleaning. However, we don't want them
new file mode 100644
--- /dev/null
+++ b/build/gen_test_backend.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozbuild.backend.test_manifest import TestManifestBackend
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader, EmptyConfig
+
+
+def gen_test_backend():
+    build_obj = MozbuildObject.from_environment()
+    try:
+        config = build_obj.config_environment
+    except BuildEnvironmentNotFoundException:
+        print("No build detected, test metadata may be incomplete.")
+        config = EmptyConfig(build_obj.topsrcdir)
+        config.topobjdir = build_obj.topobjdir
+
+    reader = BuildReader(config)
+    emitter = TreeMetadataEmitter(config)
+    backend = TestManifestBackend(config)
+
+    context = reader.read_topsrcdir()
+    data = emitter.emit(context, emitfn=emitter._process_test_manifests)
+    backend.consume(data)
+
+
+if __name__ == '__main__':
+    sys.exit(gen_test_backend())
--- a/python/moz.build
+++ b/python/moz.build
@@ -18,12 +18,16 @@ SPHINX_PYTHON_PACKAGE_DIRS += [
     'mozversioncontrol/mozversioncontrol',
 ]
 
 SPHINX_TREES['mach'] = 'mach/docs'
 
 PYTHON_UNITTEST_MANIFESTS += [
     'mach/mach/test/python.ini',
     'mozbuild/dumbmake/test/python.ini',
-    'mozbuild/mozbuild/test/python.ini',
-    'mozbuild/mozpack/test/python.ini',
     'mozlint/test/python.ini',
 ]
+
+if CONFIG['MOZ_BUILD_APP']:
+    PYTHON_UNITTEST_MANIFESTS += [
+        'mozbuild/mozbuild/test/python.ini',
+        'mozbuild/mozpack/test/python.ini',
+    ]
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -21,33 +21,33 @@ from mach.mixin.logging import LoggingMi
 import mozpack.path as mozpath
 from ..preprocessor import Preprocessor
 from ..pythonutil import iter_modules_in_path
 from ..util import (
     FileAvoidWrite,
     simple_diff,
 )
 from ..frontend.data import ContextDerived
+from ..frontend.reader import EmptyConfig
 from .configenvironment import ConfigEnvironment
 from mozbuild.base import ExecutionSummary
 
 
 class BuildBackend(LoggingMixin):
     """Abstract base class for build backends.
 
     A build backend is merely a consumer of the build configuration (the output
     of the frontend processing). It does something with said data. What exactly
     is the discretion of the specific implementation.
     """
 
     __metaclass__ = ABCMeta
 
     def __init__(self, environment):
-        assert isinstance(environment, ConfigEnvironment)
-
+        assert isinstance(environment, (ConfigEnvironment, EmptyConfig))
         self.populate_logger()
 
         self.environment = environment
 
         # Files whose modification should cause a new read and backend
         # generation.
         self.backend_input_files = set()
 
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -6,17 +6,16 @@ from __future__ import absolute_import, 
 
 import itertools
 import json
 import os
 
 import mozpack.path as mozpath
 
 from mozbuild.backend.base import BuildBackend
-from mozbuild.backend.test_manifest import TestManifestBackend
 
 from mozbuild.frontend.context import (
     Context,
     Path,
     RenamedSourcePath,
     VARIABLES,
 )
 from mozbuild.frontend.data import (
@@ -27,31 +26,28 @@ from mozbuild.frontend.data import (
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     SharedLibrary,
-    TestManifest,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
 )
 from mozbuild.jar import (
     DeprecatedJarManifest,
     JarManifestParser,
 )
 from mozbuild.preprocessor import Preprocessor
 from mozpack.chrome.manifest import parse_manifest_line
 
-from collections import defaultdict
-
 from mozbuild.util import group_unified_files
 
 class XPIDLManager(object):
     """Helps manage XPCOM IDLs in the context of the build system."""
     def __init__(self, config):
         self.config = config
         self.topsrcdir = config.topsrcdir
         self.topobjdir = config.topobjdir
@@ -179,27 +175,20 @@ class CommonBackend(BuildBackend):
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._webidls = WebIDLCollection()
         self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
 
-        # Temporarily compose a partial TestManifestBackend, soon test manifest
-        # processing will no longer be part of the build and this will be removed.
-        self._test_backend = TestManifestBackend(self.environment)
-
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
-        if isinstance(obj, TestManifest):
-            self._test_backend.consume_object(obj)
-
-        elif isinstance(obj, XPIDLFile):
+        if isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
@@ -322,19 +311,16 @@ class CommonBackend(BuildBackend):
                                                           files_per_unified_file=16))
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)
 
-        # Write out a machine-readable file describing every test.
-        self._test_backend.consume_finished()
-
         # Write out a machine-readable file describing binaries.
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -438,25 +438,22 @@ class RecursiveMakeBackend(CommonBackend
 
         if not isinstance(obj, ContextDerived):
             return False
 
         backend_file = self._get_backend_file_for(obj)
 
         consumed = CommonBackend.consume_object(self, obj)
 
-        # CommonBackend handles XPIDLFile and TestManifest, but we want to do
+        # CommonBackend handles XPIDLFile, but we want to do
         # some extra things for them.
         if isinstance(obj, XPIDLFile):
             backend_file.xpt_name = '%s.xpt' % obj.module
             self._idl_dirs.add(obj.relobjdir)
 
-        elif isinstance(obj, TestManifest):
-            self._process_test_manifest(obj, backend_file)
-
         # If CommonBackend acknowledged the object, we're done with it.
         if consumed:
             return True
 
         if not isinstance(obj, Defines):
             self.consume_object(obj.defines)
 
         if isinstance(obj, Linkable):
@@ -652,16 +649,19 @@ class RecursiveMakeBackend(CommonBackend
         elif isinstance(obj, AndroidExtraPackages):
             # Order does not matter.
             for p in sorted(set(obj.packages)):
                 backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p)
 
         elif isinstance(obj, ChromeManifestEntry):
             self._process_chrome_manifest_entry(obj, backend_file)
 
+        elif isinstance(obj, TestManifest):
+            self._process_test_manifest(obj, backend_file)
+
         else:
             return False
 
         return True
 
     def _fill_root_mk(self):
         """
         Create two files, root.mk and root-deps.mk, the first containing
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -13,16 +13,17 @@ from mozpack.manifests import (
     InstallManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import (
     RecursiveMakeBackend,
     RecursiveMakeTraversal,
 )
+from mozbuild.backend.test_manifest import TestManifestBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
 
 import mozpack.path as mozpath
 
 
@@ -555,31 +556,39 @@ class TestRecursiveMakeBackend(BackendTe
         # done.
         entries = [e for e in m._dests.keys() if '**' in e]
         self.assertEqual(len(entries), 1)
         self.assertIn('support/**', entries[0])
 
     def test_test_manifest_deffered_installs_written(self):
         """Shared support files are written to their own data file by the backend."""
         env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
+
+        # First, read the generated for ini manifest contents.
+        test_files_manifest = mozpath.join(env.topobjdir,
+                                           '_build_manifests',
+                                           'install',
+                                           '_test_files')
+        m = InstallManifest(path=test_files_manifest)
+
+        # Then, synthesize one from the test-installs.pkl file. This should
+        # allow us to re-create a subset of the above.
+        env = self._consume('test-manifest-shared-support', TestManifestBackend)
         test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
 
         with open(test_installs_path, 'r') as fh:
             test_installs = pickle.load(fh)
 
-        test_files_manifest = mozpath.join(env.topobjdir,
-                                           '_build_manifests',
-                                           'install',
-                                           '_test_files')
+        self.assertEqual(set(test_installs.keys()),
+                         set(['child/test_sub.js',
+                              'child/data/**',
+                              'child/another-file.sjs']))
+        for key in test_installs.keys():
+            self.assertIn(key, test_installs)
 
-        # First, read the generated for ini manifest contents.
-        m = InstallManifest(path=test_files_manifest)
-
-        # Then, synthesize one from the test-installs.pkl file. This should
-        # allow us to re-create a subset of the above.
         synthesized_manifest = InstallManifest()
         for item, installs in test_installs.items():
             for install_info in installs:
                 if len(install_info) == 3:
                     synthesized_manifest.add_pattern_symlink(*install_info)
                 if len(install_info) == 2:
                     synthesized_manifest.add_symlink(*install_info)
 
@@ -936,24 +945,16 @@ class TestRecursiveMakeBackend(BackendTe
             # Destination and install manifest are relative to topobjdir.
             stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
             self.assertIn(command_template % (stem, stem), lines)
 
     def test_install_manifests_package_tests(self):
         """Ensure test suites honor package_tests=False."""
         env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
-        self.assertTrue(os.path.exists(all_tests_path))
-
-        with open(all_tests_path, 'rb') as fh:
-            o = pickle.load(fh)
-            self.assertIn('mochitest.js', o)
-            self.assertIn('not_packaged.java', o)
-
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
         full = mozpath.join(man_dir, '_test_files')
         self.assertTrue(os.path.exists(full))
 
         m = InstallManifest(path=full)
 
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -180,18 +180,26 @@ class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        self._run_make(target='run-tests-deps', pass_thru=True,
-                       print_directory=False)
+        self._run_make(
+            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
+            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
+            append_env={
+                b'PYTHON': self.virtualenv_manager.python_path,
+                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
+                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
+                    self.topsrcdir, 'build', 'gen_test_backend.py'),
+            },
+        )
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',