Bug 1312574 - Use cPickle instead of json to serialize the build system's tests database. r=gps
MozReview-Commit-ID: 9xmUtVV6SRN
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -1,14 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
+import cPickle as pickle
import itertools
import json
import os
import mozpack.path as mozpath
from mozbuild.backend.base import BuildBackend
@@ -365,29 +366,28 @@ class CommonBackend(BuildBackend):
self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
for config in self._configs:
self.backend_input_files.add(config.source)
# Write out a machine-readable file describing every test.
topobjdir = self.environment.topobjdir
- with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
- json.dump(self._test_manager.tests_by_path, fh)
+ with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh:
+ pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2)
- with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh:
- json.dump(self._test_manager.manifest_defaults, fh)
+ with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh:
+ pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2)
- path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
- with self._write_file(path) as fh:
- json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
- if k in self._test_manager.deferred_installs},
- fh,
- sort_keys=True,
- indent=4)
+ path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl')
+ with self._write_file(path, mode='rb') as fh:
+ pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+ if k in self._test_manager.deferred_installs},
+ fh,
+ protocol=2)
# Write out a machine-readable file describing binaries.
with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
d = {
'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
'programs': [p.to_dict() for p in self._binaries.programs],
}
json.dump(d, fh, sort_keys=True, indent=4)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -1,14 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
+import cPickle as pickle
import json
import os
import unittest
from mozpack.manifests import (
InstallManifest,
)
from mozunit import main
@@ -519,21 +520,21 @@ class TestRecursiveMakeBackend(BackendTe
lines = [l.strip() for l in open(x_master, 'rt').readlines()]
self.assertEqual(lines, [
'; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
'',
'[include:dir1/xpcshell.ini]',
'[include:xpcshell.ini]',
])
- all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+ all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
self.assertTrue(os.path.exists(all_tests_path))
- with open(all_tests_path, 'rt') as fh:
- o = json.load(fh)
+ with open(all_tests_path, 'rb') as fh:
+ o = pickle.load(fh)
self.assertIn('xpcshell.js', o)
self.assertIn('dir1/test_bar.js', o)
self.assertEqual(len(o['xpcshell.js']), 1)
def test_test_manifest_pattern_matches_recorded(self):
"""Pattern matches in test manifests' support-files should be recorded."""
@@ -545,39 +546,39 @@ class TestRecursiveMakeBackend(BackendTe
# done.
entries = [e for e in m._dests.keys() if '**' in e]
self.assertEqual(len(entries), 1)
self.assertIn('support/**', entries[0])
def test_test_manifest_deffered_installs_written(self):
"""Shared support files are written to their own data file by the backend."""
env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
- all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+ all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
self.assertTrue(os.path.exists(all_tests_path))
- test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
+ test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
with open(test_installs_path, 'r') as fh:
- test_installs = json.load(fh)
+ test_installs = pickle.load(fh)
self.assertEqual(set(test_installs.keys()),
set(['child/test_sub.js',
'child/data/**',
'child/another-file.sjs']))
for key in test_installs.keys():
self.assertIn(key, test_installs)
test_files_manifest = mozpath.join(env.topobjdir,
'_build_manifests',
'install',
'_test_files')
# First, read the generated for ini manifest contents.
m = InstallManifest(path=test_files_manifest)
- # Then, synthesize one from the test-installs.json file. This should
+ # Then, synthesize one from the test-installs.pkl file. This should
# allow us to re-create a subset of the above.
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
synthesized_manifest.add_pattern_symlink(*install_info)
if len(install_info) == 2:
synthesized_manifest.add_symlink(*install_info)
@@ -845,21 +846,21 @@ class TestRecursiveMakeBackend(BackendTe
# Destination and install manifest are relative to topobjdir.
stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name)
self.assertIn(command_template % (stem, stem), lines)
def test_install_manifests_package_tests(self):
"""Ensure test suites honor package_tests=False."""
env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
- all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+ all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
self.assertTrue(os.path.exists(all_tests_path))
- with open(all_tests_path, 'rt') as fh:
- o = json.load(fh)
+ with open(all_tests_path, 'rb') as fh:
+ o = pickle.load(fh)
self.assertIn('mochitest.js', o)
self.assertIn('not_packaged.java', o)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
full = mozpath.join(man_dir, '_test_files')
self.assertTrue(os.path.exists(full))
--- a/python/mozbuild/mozbuild/test/test_testing.py
+++ b/python/mozbuild/mozbuild/test/test_testing.py
@@ -1,14 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
+import cPickle as pickle
import os
import shutil
import tempfile
import unittest
import mozpack.path as mozpath
from mozfile import NamedTemporaryFile
@@ -16,18 +17,17 @@ from mozunit import main
from mozbuild.base import MozbuildObject
from mozbuild.testing import (
TestMetadata,
TestResolver,
)
-ALL_TESTS_JSON = b'''
-{
+ALL_TESTS = {
"accessible/tests/mochitest/actions/test_anchors.html": [
{
"dir_relpath": "accessible/tests/mochitest/actions",
"expected": "pass",
"file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
"flavor": "a11y",
"here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions",
"manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini",
@@ -149,41 +149,41 @@ ALL_TESTS_JSON = b'''
"manifest": "/home/chris/m-c/devtools/client/markupview/test/browser.ini",
"name": "browser_markupview_copy_image_data.js",
"path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js",
"relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js",
"subsuite": "devtools",
"tags": "devtools"
}
]
-}'''.strip()
+}
-TEST_DEFAULTS = b'''{
- "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"}
-}'''
+TEST_DEFAULTS = {
+ "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
+}
class Base(unittest.TestCase):
def setUp(self):
self._temp_files = []
def tearDown(self):
for f in self._temp_files:
del f
self._temp_files = []
def _get_test_metadata(self):
- all_tests = NamedTemporaryFile()
- all_tests.write(ALL_TESTS_JSON)
+ all_tests = NamedTemporaryFile(mode='wb')
+ pickle.dump(ALL_TESTS, all_tests)
all_tests.flush()
self._temp_files.append(all_tests)
- test_defaults = NamedTemporaryFile()
- test_defaults.write(TEST_DEFAULTS)
+ test_defaults = NamedTemporaryFile(mode='wb')
+ pickle.dump(TEST_DEFAULTS, test_defaults)
test_defaults.flush()
self._temp_files.append(test_defaults)
return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
class TestTestMetadata(Base):
def test_load(self):
@@ -246,20 +246,20 @@ class TestTestResolver(Base):
for d in self._temp_dirs:
shutil.rmtree(d)
def _get_resolver(self):
topobjdir = tempfile.mkdtemp()
self._temp_dirs.append(topobjdir)
- with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
- fh.write(ALL_TESTS_JSON)
- with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh:
- fh.write(TEST_DEFAULTS)
+ with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
+ pickle.dump(ALL_TESTS, fh)
+ with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
+ pickle.dump(TEST_DEFAULTS, fh)
o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
# Monkey patch the test resolver to avoid tests failing to find make
# due to our fake topscrdir.
TestResolver._run_make = lambda *a, **b: None
return o._spawn(TestResolver)
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -1,15 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
-import json
+import cPickle as pickle
import os
import sys
import mozpack.path as mozpath
from mozpack.copier import FileCopier
from mozpack.manifests import InstallManifest
@@ -51,22 +51,22 @@ class TestMetadata(object):
configuration.
"""
def __init__(self, all_tests, test_defaults=None):
self._tests_by_path = OrderedDefaultDict(list)
self._tests_by_flavor = defaultdict(set)
self._test_dirs = set()
- with open(all_tests, 'rt') as fh:
- test_data = json.load(fh)
+ with open(all_tests, 'rb') as fh:
+ test_data = pickle.load(fh)
defaults = None
if test_defaults:
- with open(test_defaults, 'rt') as fh:
- defaults = json.load(fh)
+ with open(test_defaults, 'rb') as fh:
+ defaults = pickle.load(fh)
for path, tests in test_data.items():
for metadata in tests:
if defaults:
manifest = metadata['manifest']
manifest_defaults = defaults.get(manifest)
if manifest_defaults:
metadata = manifestparser.combine_fields(manifest_defaults,
metadata)
@@ -173,24 +173,24 @@ class TestMetadata(object):
class TestResolver(MozbuildObject):
"""Helper to resolve tests from the current environment to test files."""
def __init__(self, *args, **kwargs):
MozbuildObject.__init__(self, *args, **kwargs)
# If installing tests is going to result in re-generating the build
# backend, we need to do this here, so that the updated contents of
- # all-tests.json make it to the set of tests to run.
+ # all-tests.pkl make it to the set of tests to run.
self._run_make(target='run-tests-deps', pass_thru=True,
print_directory=False)
self._tests = TestMetadata(os.path.join(self.topobjdir,
- 'all-tests.json'),
+ 'all-tests.pkl'),
test_defaults=os.path.join(self.topobjdir,
- 'test-defaults.json'))
+ 'test-defaults.pkl'))
self._test_rewrites = {
'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
'mochitest', 'a11y'),
'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
'mochitest', 'browser'),
'jetpack-package': os.path.join(self.topobjdir, '_tests', 'testing',
'mochitest', 'jetpack-package'),
@@ -412,19 +412,19 @@ class SupportFilesConverter(object):
info.installs.append((full, mozpath.normpath(dest_path)))
return info
def _resolve_installs(paths, topobjdir, manifest):
"""Using the given paths as keys, find any unresolved installs noted
by the build backend corresponding to those keys, and add them
to the given manifest.
"""
- filename = os.path.join(topobjdir, 'test-installs.json')
- with open(filename, 'r') as fh:
- resolved_installs = json.load(fh)
+ filename = os.path.join(topobjdir, 'test-installs.pkl')
+ with open(filename, 'rb') as fh:
+ resolved_installs = pickle.load(fh)
for path in paths:
path = path[2:]
if path not in resolved_installs:
raise Exception('A cross-directory support file path noted in a '
'test manifest does not appear in any other manifest.\n "%s" '
'must appear in another test manifest to specify an install '
'for "!/%s".' % (path, path))