Bug 1320194 - Generate all-tests.pkl and related files when resolving tests
This replaces the 'run-tests-deps' make target with a python function that will directly
read moz.build files, emit them with TestManifestEmitter, then consume them with
TestManifestBackend. Because the TestResolver is the only place that actually reads the
test metadata files, we can remove this logic from the CommonBackend as well.
MozReview-Commit-ID: DXgMoeH5dKf
MozReview-Commit-ID: HstZ57qkqf2
--- a/Makefile.in
+++ b/Makefile.in
@@ -181,21 +181,16 @@ endif
$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
# Dummy wrapper rule to allow the faster backend to piggy back
$(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
.PHONY: install-tests
install-tests: install-test-files
-# We no longer run "make install-tests" directly before running tests, but we still
-# want to depend on things like config.status, hence this target.
-.PHONY: run-tests-deps
-run-tests-deps: $(install_manifest_depends)
-
# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
.PHONY: install-test-files
install-test-files:
$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
include $(topsrcdir)/build/moz-automation.mk
# dist and _tests should be purged during cleaning. However, we don't want them
new file mode 100644
--- /dev/null
+++ b/build/gen_test_backend.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from mozbuild.backend.test_manifest import TestManifestBackend
+from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader, EmptyConfig
+
+
+def gen_test_backend():
+ build_obj = MozbuildObject.from_environment()
+ try:
+ config = build_obj.config_environment
+ except BuildEnvironmentNotFoundException:
+ print("No build detected, test metadata may be incomplete.")
+ config = EmptyConfig(build_obj.topsrcdir)
+ config.topobjdir = build_obj.topobjdir
+
+ reader = BuildReader(config)
+ emitter = TreeMetadataEmitter(config)
+ backend = TestManifestBackend(config)
+
+ context = reader.read_topsrcdir()
+ data = emitter.emit(context, emitfn=emitter._process_test_manifests)
+ backend.consume(data)
+
+
+if __name__ == '__main__':
+ sys.exit(gen_test_backend())
--- a/python/moz.build
+++ b/python/moz.build
@@ -18,12 +18,16 @@ SPHINX_PYTHON_PACKAGE_DIRS += [
'mozversioncontrol/mozversioncontrol',
]
SPHINX_TREES['mach'] = 'mach/docs'
PYTHON_UNITTEST_MANIFESTS += [
'mach/mach/test/python.ini',
'mozbuild/dumbmake/test/python.ini',
- 'mozbuild/mozbuild/test/python.ini',
- 'mozbuild/mozpack/test/python.ini',
'mozlint/test/python.ini',
]
+
+if CONFIG['MOZ_BUILD_APP']:
+ PYTHON_UNITTEST_MANIFESTS += [
+ 'mozbuild/mozbuild/test/python.ini',
+ 'mozbuild/mozpack/test/python.ini',
+ ]
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -21,33 +21,33 @@ from mach.mixin.logging import LoggingMi
import mozpack.path as mozpath
from ..preprocessor import Preprocessor
from ..pythonutil import iter_modules_in_path
from ..util import (
FileAvoidWrite,
simple_diff,
)
from ..frontend.data import ContextDerived
+from ..frontend.reader import EmptyConfig
from .configenvironment import ConfigEnvironment
from mozbuild.base import ExecutionSummary
class BuildBackend(LoggingMixin):
"""Abstract base class for build backends.
A build backend is merely a consumer of the build configuration (the output
of the frontend processing). It does something with said data. What exactly
is the discretion of the specific implementation.
"""
__metaclass__ = ABCMeta
def __init__(self, environment):
- assert isinstance(environment, ConfigEnvironment)
-
+ assert isinstance(environment, (ConfigEnvironment, EmptyConfig))
self.populate_logger()
self.environment = environment
# Files whose modification should cause a new read and backend
# generation.
self.backend_input_files = set()
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -6,17 +6,16 @@ from __future__ import absolute_import,
import itertools
import json
import os
import mozpack.path as mozpath
from mozbuild.backend.base import BuildBackend
-from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.frontend.context import (
Context,
Path,
RenamedSourcePath,
VARIABLES,
)
from mozbuild.frontend.data import (
@@ -27,31 +26,28 @@ from mozbuild.frontend.data import (
IPDLFile,
FinalTargetPreprocessedFiles,
FinalTargetFiles,
GeneratedEventWebIDLFile,
GeneratedWebIDLFile,
PreprocessedTestWebIDLFile,
PreprocessedWebIDLFile,
SharedLibrary,
- TestManifest,
TestWebIDLFile,
UnifiedSources,
XPIDLFile,
WebIDLFile,
)
from mozbuild.jar import (
DeprecatedJarManifest,
JarManifestParser,
)
from mozbuild.preprocessor import Preprocessor
from mozpack.chrome.manifest import parse_manifest_line
-from collections import defaultdict
-
from mozbuild.util import group_unified_files
class XPIDLManager(object):
"""Helps manage XPCOM IDLs in the context of the build system."""
def __init__(self, config):
self.config = config
self.topsrcdir = config.topsrcdir
self.topobjdir = config.topobjdir
@@ -179,27 +175,20 @@ class CommonBackend(BuildBackend):
def _init(self):
self._idl_manager = XPIDLManager(self.environment)
self._webidls = WebIDLCollection()
self._binaries = BinariesCollection()
self._configs = set()
self._ipdl_sources = set()
- # Temporarily compose a partial TestManifestBackend, soon test manifest
- # processing will no longer be part of the build and this will be removed.
- self._test_backend = TestManifestBackend(self.environment)
-
def consume_object(self, obj):
self._configs.add(obj.config)
- if isinstance(obj, TestManifest):
- self._test_backend.consume_object(obj)
-
- elif isinstance(obj, XPIDLFile):
+ if isinstance(obj, XPIDLFile):
# TODO bug 1240134 tracks not processing XPIDL files during
# artifact builds.
self._idl_manager.register_idl(obj)
elif isinstance(obj, ConfigFileSubstitution):
# Do not handle ConfigFileSubstitution for Makefiles. Leave that
# to other
if mozpath.basename(obj.output_path) == 'Makefile':
@@ -322,19 +311,16 @@ class CommonBackend(BuildBackend):
files_per_unified_file=16))
self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
for config in self._configs:
self.backend_input_files.add(config.source)
- # Write out a machine-readable file describing every test.
- self._test_backend.consume_finished()
-
# Write out a machine-readable file describing binaries.
topobjdir = self.environment.topobjdir
with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
d = {
'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
'programs': [p.to_dict() for p in self._binaries.programs],
}
json.dump(d, fh, sort_keys=True, indent=4)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -438,25 +438,22 @@ class RecursiveMakeBackend(CommonBackend
if not isinstance(obj, ContextDerived):
return False
backend_file = self._get_backend_file_for(obj)
consumed = CommonBackend.consume_object(self, obj)
- # CommonBackend handles XPIDLFile and TestManifest, but we want to do
+ # CommonBackend handles XPIDLFile, but we want to do
# some extra things for them.
if isinstance(obj, XPIDLFile):
backend_file.xpt_name = '%s.xpt' % obj.module
self._idl_dirs.add(obj.relobjdir)
- elif isinstance(obj, TestManifest):
- self._process_test_manifest(obj, backend_file)
-
# If CommonBackend acknowledged the object, we're done with it.
if consumed:
return True
if not isinstance(obj, Defines):
self.consume_object(obj.defines)
if isinstance(obj, Linkable):
@@ -652,16 +649,19 @@ class RecursiveMakeBackend(CommonBackend
elif isinstance(obj, AndroidExtraPackages):
# Order does not matter.
for p in sorted(set(obj.packages)):
backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p)
elif isinstance(obj, ChromeManifestEntry):
self._process_chrome_manifest_entry(obj, backend_file)
+ elif isinstance(obj, TestManifest):
+ self._process_test_manifest(obj, backend_file)
+
else:
return False
return True
def _fill_root_mk(self):
"""
Create two files, root.mk and root-deps.mk, the first containing
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -13,16 +13,17 @@ from mozpack.manifests import (
InstallManifest,
)
from mozunit import main
from mozbuild.backend.recursivemake import (
RecursiveMakeBackend,
RecursiveMakeTraversal,
)
+from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader
from mozbuild.test.backend.common import BackendTester
import mozpack.path as mozpath
@@ -555,31 +556,39 @@ class TestRecursiveMakeBackend(BackendTe
# done.
entries = [e for e in m._dests.keys() if '**' in e]
self.assertEqual(len(entries), 1)
self.assertIn('support/**', entries[0])
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
+
+ # First, read the generated for ini manifest contents.
+ test_files_manifest = mozpath.join(env.topobjdir,
+ '_build_manifests',
+ 'install',
+ '_test_files')
+ m = InstallManifest(path=test_files_manifest)
+
+ # Then, synthesize one from the test-installs.pkl file. This should
+ # allow us to re-create a subset of the above.
+ env = self._consume('test-manifest-shared-support', TestManifestBackend)
test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
with open(test_installs_path, 'r') as fh:
test_installs = pickle.load(fh)
- test_files_manifest = mozpath.join(env.topobjdir,
- '_build_manifests',
- 'install',
- '_test_files')
+ self.assertEqual(set(test_installs.keys()),
+ set(['child/test_sub.js',
+ 'child/data/**',
+ 'child/another-file.sjs']))
+ for key in test_installs.keys():
+ self.assertIn(key, test_installs)
- # First, read the generated for ini manifest contents.
- m = InstallManifest(path=test_files_manifest)
-
- # Then, synthesize one from the test-installs.pkl file. This should
- # allow us to re-create a subset of the above.
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
synthesized_manifest.add_pattern_symlink(*install_info)
if len(install_info) == 2:
synthesized_manifest.add_symlink(*install_info)
@@ -936,24 +945,16 @@ class TestRecursiveMakeBackend(BackendTe
# Destination and install manifest are relative to topobjdir.
stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
self.assertIn(command_template % (stem, stem), lines)
def test_install_manifests_package_tests(self):
"""Ensure test suites honor package_tests=False."""
env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
- all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
- self.assertTrue(os.path.exists(all_tests_path))
-
- with open(all_tests_path, 'rb') as fh:
- o = pickle.load(fh)
- self.assertIn('mochitest.js', o)
- self.assertIn('not_packaged.java', o)
-
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
full = mozpath.join(man_dir, '_test_files')
self.assertTrue(os.path.exists(full))
m = InstallManifest(path=full)
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -180,18 +180,26 @@ class TestResolver(MozbuildObject):
"""Helper to resolve tests from the current environment to test files."""
def __init__(self, *args, **kwargs):
MozbuildObject.__init__(self, *args, **kwargs)
# If installing tests is going to result in re-generating the build
# backend, we need to do this here, so that the updated contents of
# all-tests.pkl make it to the set of tests to run.
- self._run_make(target='run-tests-deps', pass_thru=True,
- print_directory=False)
+ self._run_make(
+ target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
+ filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
+ append_env={
+ b'PYTHON': self.virtualenv_manager.python_path,
+ b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
+ b'BACKEND_GENERATION_SCRIPT': mozpath.join(
+ self.topsrcdir, 'build', 'gen_test_backend.py'),
+ },
+ )
self._tests = TestMetadata(os.path.join(self.topobjdir,
'all-tests.pkl'),
test_defaults=os.path.join(self.topobjdir,
'test-defaults.pkl'))
self._test_rewrites = {
'a11y': os.path.join(self.topobjdir, '_tests', 'testing',