Bug 1264697 - Change the format of all-tests.json to reduce redundant data. r=gps
The format provided to the build system by the manifest parser is highly redundant:
every test lists all variables for that test, and many tests use a large
support-files entry in DEFAULT that ends up in individual test objects. This
patch stores these DEFAULTS per-manifest rather than per-test to save disk
space, resulting in about a ~22mb smaller all-tests.json file. The
in-memory representation of tests is not changed by this patch, as the defaults
are again propagated to individual tests as all-tests.json is read by the test
resolver.
MozReview-Commit-ID: CEJaevfS5s7
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -171,28 +171,40 @@ class TestManager(object):
def __init__(self, config):
self.config = config
self.topsrcdir = mozpath.normpath(config.topsrcdir)
self.tests_by_path = defaultdict(list)
self.installs_by_path = defaultdict(list)
self.deferred_installs = set()
+ self.manifest_default_support_files = {}
- def add(self, t, flavor, topsrcdir):
+ def add(self, t, flavor, topsrcdir, default_supp_files):
t = dict(t)
t['flavor'] = flavor
path = mozpath.normpath(t['path'])
assert mozpath.basedir(path, [topsrcdir])
key = path[len(topsrcdir)+1:]
t['file_relpath'] = key
t['dir_relpath'] = mozpath.dirname(key)
+ # Support files are propagated from the default section to individual
+ # tests by the manifest parser, but we end up storing a lot of
+ # redundant data due to the huge number of support files.
+ # So if we have support files that are the same as the manifest default
+ # we track that separately, per-manifest instead of per-test, to save
+ # space.
+ supp_files = t.get('support-files')
+ if supp_files and supp_files == default_supp_files:
+ self.manifest_default_support_files[t['manifest']] = default_supp_files
+ del t['support-files']
+
self.tests_by_path[key].append(t)
def add_installs(self, obj, topsrcdir):
for src, (dest, _) in obj.installs.iteritems():
key = src[len(topsrcdir)+1:]
self.installs_by_path[key].append((src, dest))
for src, pat, dest in obj.pattern_installs:
key = mozpath.join(src[len(topsrcdir)+1:], pat)
@@ -220,17 +232,18 @@ class CommonBackend(BuildBackend):
self._configs = set()
self._ipdl_sources = set()
def consume_object(self, obj):
self._configs.add(obj.config)
if isinstance(obj, TestManifest):
for test in obj.tests:
- self._test_manager.add(test, obj.flavor, obj.topsrcdir)
+ self._test_manager.add(test, obj.flavor, obj.topsrcdir,
+ obj.default_support_files)
self._test_manager.add_installs(obj, obj.topsrcdir)
elif isinstance(obj, XPIDLFile):
# TODO bug 1240134 tracks not processing XPIDL files during
# artifact builds.
self._idl_manager.register_idl(obj)
elif isinstance(obj, ConfigFileSubstitution):
@@ -359,17 +372,18 @@ class CommonBackend(BuildBackend):
self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
for config in self._configs:
self.backend_input_files.add(config.source)
# Write out a machine-readable file describing every test.
topobjdir = self.environment.topobjdir
with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
- json.dump(self._test_manager.tests_by_path, fh)
+ json.dump([self._test_manager.tests_by_path,
+ self._test_manager.manifest_default_support_files], fh)
path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
with self._write_file(path) as fh:
json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
if k in self._test_manager.deferred_installs},
fh,
sort_keys=True,
indent=4)
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -581,16 +581,21 @@ class TestManifest(ContextDerived):
'manifest_relpath',
# The relative path of the parsed manifest within the objdir.
'manifest_obj_relpath',
# If this manifest is a duplicate of another one, this is the
# manifestparser.TestManifest of the other one.
'dupe_manifest',
+
+ # The support files appearing in the DEFAULT section of this
+ # manifest. This enables a space optimization in all-tests.json,
+ # see the comment in mozbuild/backend/common.py.
+ 'default_support_files',
)
def __init__(self, context, path, manifest, flavor=None,
install_prefix=None, relpath=None, dupe_manifest=False):
ContextDerived.__init__(self, context)
assert flavor in all_test_flavors()
@@ -602,16 +607,17 @@ class TestManifest(ContextDerived):
self.manifest_relpath = relpath
self.manifest_obj_relpath = relpath
self.dupe_manifest = dupe_manifest
self.installs = {}
self.pattern_installs = []
self.tests = []
self.external_installs = set()
self.deferred_installs = set()
+ self.default_support_files = None
class LocalInclude(ContextDerived):
"""Describes an individual local include path."""
__slots__ = (
'path',
)
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1075,16 +1075,20 @@ class TreeMetadataEmitter(LoggingMixin):
raise SandboxValidationError('Empty test manifest: %s'
% path, context)
obj = TestManifest(context, path, mpmanifest, flavor=flavor,
install_prefix=install_prefix,
relpath=mozpath.join(manifest_reldir, mozpath.basename(path)),
dupe_manifest='dupe-manifest' in defaults)
+ default_support_files = defaults.get('support-files')
+ if default_support_files:
+ obj.default_support_files = default_support_files
+
filtered = mpmanifest.tests
# Jetpack add-on tests are expected to be generated during the
# build process so they won't exist here.
if flavor != 'jetpack-addon':
missing = [t['name'] for t in filtered if not os.path.exists(t['path'])]
if missing:
raise SandboxValidationError('Test manifest (%s) lists '
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -523,17 +523,17 @@ class TestRecursiveMakeBackend(BackendTe
'[include:dir1/xpcshell.ini]',
'[include:xpcshell.ini]',
])
all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
self.assertTrue(os.path.exists(all_tests_path))
with open(all_tests_path, 'rt') as fh:
- o = json.load(fh)
+ o, _ = json.load(fh)
self.assertIn('xpcshell.js', o)
self.assertIn('dir1/test_bar.js', o)
self.assertEqual(len(o['xpcshell.js']), 1)
def test_test_manifest_pattern_matches_recorded(self):
"""Pattern matches in test manifests' support-files should be recorded."""
@@ -845,23 +845,21 @@ class TestRecursiveMakeBackend(BackendTe
# Destination and install manifest are relative to topobjdir.
stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
self.assertIn(command_template % (stem, stem), lines)
def test_install_manifests_package_tests(self):
"""Ensure test suites honor package_tests=False."""
env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
- tests_dir = mozpath.join(env.topobjdir, '_tests')
-
all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
self.assertTrue(os.path.exists(all_tests_path))
with open(all_tests_path, 'rt') as fh:
- o = json.load(fh)
+ o, _ = json.load(fh)
self.assertIn('mochitest.js', o)
self.assertIn('not_packaged.java', o)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
full = mozpath.join(man_dir, '_test_files')
--- a/python/mozbuild/mozbuild/test/test_testing.py
+++ b/python/mozbuild/mozbuild/test/test_testing.py
@@ -17,17 +17,17 @@ from mozunit import main
from mozbuild.base import MozbuildObject
from mozbuild.testing import (
TestMetadata,
TestResolver,
)
ALL_TESTS_JSON = b'''
-{
+[{
"accessible/tests/mochitest/actions/test_anchors.html": [
{
"dir_relpath": "accessible/tests/mochitest/actions",
"expected": "pass",
"file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
"flavor": "a11y",
"here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions",
"manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini",
@@ -76,34 +76,32 @@ ALL_TESTS_JSON = b'''
"here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
"manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
"name": "test_0201_app_launch_apply_update.js",
"path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
"reason": "bug 820380",
"relpath": "test_0201_app_launch_apply_update.js",
"run-sequentially": "Launches application.",
"skip-if": "toolkit == 'gonk' || os == 'android'",
- "support-files": "\\ndata/**\\nxpcshell_updater.ini",
"tail": ""
},
{
"dir_relpath": "toolkit/mozapps/update/test/unit",
"file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
"flavor": "xpcshell",
"generated-files": "head_update.js",
"head": "head_update.js head2.js",
"here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
"manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
"name": "test_0201_app_launch_apply_update.js",
"path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
"reason": "bug 820380",
"relpath": "test_0201_app_launch_apply_update.js",
"run-sequentially": "Launches application.",
"skip-if": "toolkit == 'gonk' || os == 'android'",
- "support-files": "\\ndata/**\\nxpcshell_updater.ini",
"tail": ""
}
],
"mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [
{
"dir_relpath": "mobile/android/tests/background/junit3/src/common",
"file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java",
"flavor": "instrumentation",
@@ -151,17 +149,19 @@ ALL_TESTS_JSON = b'''
"manifest": "/home/chris/m-c/devtools/client/markupview/test/browser.ini",
"name": "browser_markupview_copy_image_data.js",
"path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
"relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
"subsuite": "devtools",
"tags": "devtools"
}
]
-}'''.strip()
+}, {
+ "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": "\\ndata/**\\nxpcshell_updater.ini"
+}]'''.strip()
class Base(unittest.TestCase):
def setUp(self):
self._temp_files = []
def tearDown(self):
for f in self._temp_files:
@@ -205,16 +205,26 @@ class TestTestMetadata(Base):
self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
under_path='services'))), 2)
def test_resolve_multiple_paths(self):
t = self._get_test_metadata()
result = list(t.resolve_tests(paths=['services', 'toolkit']))
self.assertEqual(len(result), 4)
+ def test_resolve_support_files(self):
+ expected_support_files = "\ndata/**\nxpcshell_updater.ini"
+ t = self._get_test_metadata()
+ result = list(t.resolve_tests(paths=['toolkit']))
+ self.assertEqual(len(result), 2)
+
+ for test in result:
+ self.assertEqual(test['support-files'],
+ expected_support_files)
+
def test_resolve_path_prefix(self):
t = self._get_test_metadata()
result = list(t.resolve_tests(paths=['image']))
self.assertEqual(len(result), 1)
class TestTestResolver(Base):
FAKE_TOPSRCDIR = '/Users/gps/src/firefox'
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -54,20 +54,24 @@ class TestMetadata(object):
def __init__(self, filename=None):
self._tests_by_path = OrderedDefaultDict(list)
self._tests_by_flavor = defaultdict(set)
self._test_dirs = set()
if filename:
with open(filename, 'rt') as fh:
- d = json.load(fh)
+ test_data, manifest_support_files = json.load(fh)
- for path, tests in d.items():
+ for path, tests in test_data.items():
for metadata in tests:
+ manifest = metadata['manifest']
+ support_files = manifest_support_files.get(manifest)
+ if support_files and 'support-files' not in metadata:
+ metadata['support-files'] = support_files
self._tests_by_path[path].append(metadata)
self._test_dirs.add(os.path.dirname(path))
flavor = metadata.get('flavor')
self._tests_by_flavor[flavor].add(path)
def tests_with_flavor(self, flavor):
"""Obtain all tests having the specified flavor.