--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -21,26 +21,34 @@ import os
from collections import (
Counter,
OrderedDict,
)
from mozbuild.util import (
HierarchicalStringList,
KeyedDefaultDict,
List,
+ ListWithAction,
memoize,
memoized_property,
ReadOnlyKeyedDefaultDict,
StrictOrderingOnAppendList,
+ StrictOrderingOnAppendListWithAction,
StrictOrderingOnAppendListWithFlagsFactory,
TypedList,
TypedNamedTuple,
)
-from ..testing import all_test_flavors
+from ..testing import (
+ all_test_flavors,
+ read_manifestparser_manifest,
+ read_reftest_manifest,
+ read_wpt_manifest,
+)
+
import mozpack.path as mozpath
from types import FunctionType
import itertools
class ContextDerivedValue(object):
"""Classes deriving from this one receive a special treatment in a
@@ -74,25 +82,26 @@ class Context(KeyedDefaultDict):
this context instance. Keys in this dict are the strings representing keys
in this context which are valid. Values are tuples of stored type,
assigned type, default value, a docstring describing the purpose of the
variable, and a tier indicator (see comment above the VARIABLES declaration
in this module).
config is the ConfigEnvironment for this context.
"""
- def __init__(self, allowed_variables={}, config=None):
+ def __init__(self, allowed_variables={}, config=None, finder=None):
self._allowed_variables = allowed_variables
self.main_path = None
self.current_path = None
# There aren't going to be enough paths for the performance of scanning
# a list to be a problem.
self._all_paths = []
self.config = config
self._sandbox = None
+ self._finder = finder
KeyedDefaultDict.__init__(self, self._factory)
def push_source(self, path):
"""Adds the given path as source of the data from this context and make
it the current path for the context."""
assert os.path.isabs(path)
if not self.main_path:
self.main_path = path
@@ -567,31 +576,65 @@ def ContextDerivedTypedHierarchicalStrin
child = self._children.get(name)
if not child:
child = self._children[name] = _TypedListWithItems(
self._context)
return child
return _TypedListWithItems
-
-BugzillaComponent = TypedNamedTuple('BugzillaComponent',
- [('product', unicode), ('component', unicode)])
+def OrderedListWithAction(action):
+ """Returns a class which behaves as a StrictOrderingOnAppendList, but
+ invokes the given given callable with each input and a context as it is
+ read, storing a tuple including the result and the original item.
+
+ This used to extend moz.build reading to make more data available in
+ filesystem-reading mode.
+ """
+ class _OrderedListWithAction(ContextDerivedValue,
+ StrictOrderingOnAppendListWithAction):
+ def __init__(self, context, *args):
+ def _action(item):
+ return item, action(context, item)
+ super(_OrderedListWithAction, self).__init__(action=_action, *args)
+
+ return _OrderedListWithAction
+
+def TypedListWithAction(typ, action):
+ """Returns a class which behaves as a TypedList with the provided type, but
+ invokes the given given callable with each input and a context as it is
+ read, storing a tuple including the result and the original item.
+
+ This used to extend moz.build reading to make more data available in
+ filesystem-reading mode.
+ """
+ class _TypedListWithAction(ContextDerivedValue, TypedList(typ), ListWithAction):
+ def __init__(self, context, *args):
+ def _action(item):
+ return item, action(context, item)
+ super(_TypedListWithAction, self).__init__(action=_action, *args)
+ return _TypedListWithAction
WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest",
[("manifest_path", unicode),
("test_root", unicode)])
+ManifestparserManifestList = OrderedListWithAction(read_manifestparser_manifest)
+ReftestManifestList = OrderedListWithAction(read_reftest_manifest)
+WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest)
OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList)
OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
StrictOrderingOnAppendList)
OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList)
DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
('tags', OrderedStringList),
('flavors', OrderedTestFlavorList))
+BugzillaComponent = TypedNamedTuple('BugzillaComponent',
+ [('product', unicode), ('component', unicode)])
+
class Files(SubContext):
"""Metadata attached to files.
It is common to want to annotate files with metadata, such as which
Bugzilla component tracks issues with certain files. This sub-context is
where we stick that metadata.
@@ -1419,97 +1462,97 @@ VARIABLES = {
"""Names of example WebIDL interfaces to build as part of the build.
Names in this list correspond to WebIDL interface names defined in
WebIDL files included in the build from one of the \*WEBIDL_FILES
variables.
"""),
# Test declaration.
- 'A11Y_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'A11Y_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining a11y tests.
"""),
- 'BROWSER_CHROME_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'BROWSER_CHROME_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining browser chrome tests.
"""),
- 'JETPACK_PACKAGE_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'JETPACK_PACKAGE_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining jetpack package tests.
"""),
- 'JETPACK_ADDON_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'JETPACK_ADDON_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining jetpack addon tests.
"""),
- 'CRASHTEST_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'ANDROID_INSTRUMENTATION_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining Android instrumentation tests.
+ """),
+
+ 'MARIONETTE_LAYOUT_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining marionette-layout tests.
+ """),
+
+ 'MARIONETTE_LOOP_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining marionette-loop tests.
+ """),
+
+ 'MARIONETTE_UNIT_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining marionette-unit tests.
+ """),
+
+ 'MARIONETTE_UPDATE_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining marionette-update tests.
+ """),
+
+ 'MARIONETTE_WEBAPI_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining marionette-webapi tests.
+ """),
+
+ 'METRO_CHROME_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining metro browser chrome tests.
+ """),
+
+ 'MOCHITEST_CHROME_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining mochitest chrome tests.
+ """),
+
+ 'MOCHITEST_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining mochitest tests.
+ """),
+
+ 'MOCHITEST_WEBAPPRT_CONTENT_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining webapprt mochitest content tests.
+ """),
+
+ 'MOCHITEST_WEBAPPRT_CHROME_MANIFESTS': (ManifestparserManifestList, list,
+ """List of manifest files defining webapprt mochitest chrome tests.
+ """),
+
+ 'REFTEST_MANIFESTS': (ReftestManifestList, list,
+ """List of manifest files defining reftests.
+
+ These are commonly named reftest.list.
+ """),
+
+ 'CRASHTEST_MANIFESTS': (ReftestManifestList, list,
"""List of manifest files defining crashtests.
These are commonly named crashtests.list.
"""),
- 'ANDROID_INSTRUMENTATION_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining Android instrumentation tests.
- """),
-
- 'MARIONETTE_LAYOUT_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining marionette-layout tests.
- """),
-
- 'MARIONETTE_LOOP_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining marionette-loop tests.
- """),
-
- 'MARIONETTE_UNIT_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining marionette-unit tests.
- """),
-
- 'MARIONETTE_UPDATE_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining marionette-update tests.
- """),
-
- 'MARIONETTE_WEBAPI_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining marionette-webapi tests.
- """),
-
- 'METRO_CHROME_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining metro browser chrome tests.
- """),
-
- 'MOCHITEST_CHROME_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining mochitest chrome tests.
- """),
-
- 'MOCHITEST_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining mochitest tests.
- """),
-
- 'MOCHITEST_WEBAPPRT_CONTENT_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining webapprt mochitest content tests.
- """),
-
- 'MOCHITEST_WEBAPPRT_CHROME_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining webapprt mochitest chrome tests.
- """),
-
- 'REFTEST_MANIFESTS': (StrictOrderingOnAppendList, list,
- """List of manifest files defining reftests.
-
- These are commonly named reftest.list.
- """),
-
- 'WEB_PLATFORM_TESTS_MANIFESTS': (TypedList(WebPlatformTestManifest), list,
+ 'WEB_PLATFORM_TESTS_MANIFESTS': (WptManifestList, list,
"""List of (manifest_path, test_path) defining web-platform-tests.
"""),
- 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining WebRTC signalling tests.
"""),
- 'XPCSHELL_TESTS_MANIFESTS': (StrictOrderingOnAppendList, list,
+ 'XPCSHELL_TESTS_MANIFESTS': (ManifestparserManifestList, list,
"""List of manifest files defining xpcshell tests.
"""),
# The following variables are used to control the target of installed files.
'XPI_NAME': (unicode, unicode,
"""The name of an extension XPI to generate.
When this variable is present, the results of this directory will end up
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -14,18 +14,16 @@ import time
from collections import defaultdict, OrderedDict
from mach.mixin.logging import LoggingMixin
from mozbuild.util import (
memoize,
OrderedDefaultDict,
)
import mozpack.path as mozpath
-import manifestparser
-import reftest
import mozinfo
from .data import (
AndroidAssetsDirs,
AndroidExtraPackages,
AndroidExtraResDirs,
AndroidResDirs,
BrandingFiles,
@@ -49,17 +47,16 @@ from .data import (
HostProgram,
HostSimpleProgram,
HostSources,
InstallationTarget,
IPDLFile,
JARManifest,
Library,
Linkable,
- LinkageWrongKindError,
LocalInclude,
PerSourceFlag,
PreprocessedTestWebIDLFile,
PreprocessedWebIDLFile,
Program,
SharedLibrary,
SimpleProgram,
Sources,
@@ -1056,54 +1053,51 @@ class TreeMetadataEmitter(LoggingMixin):
else:
srcdir_files[path].append(p.full_path)
yield TestHarnessFiles(context, srcdir_files,
srcdir_pattern_files, objdir_files)
def _process_test_manifests(self, context):
for prefix, info in TEST_MANIFESTS.items():
- for path in context.get('%s_MANIFESTS' % prefix, []):
- for obj in self._process_test_manifest(context, info, path):
+ for path, manifest in context.get('%s_MANIFESTS' % prefix, []):
+ for obj in self._process_test_manifest(context, info, path, manifest):
yield obj
for flavor in REFTEST_FLAVORS:
- for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
- for obj in self._process_reftest_manifest(context, flavor, path):
+ for path, manifest in context.get('%s_MANIFESTS' % flavor.upper(), []):
+ for obj in self._process_reftest_manifest(context, flavor, path, manifest):
yield obj
for flavor in WEB_PATFORM_TESTS_FLAVORS:
- for path in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
- for obj in self._process_web_platform_tests_manifest(context, path):
+ for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
+ for obj in self._process_web_platform_tests_manifest(context, path, manifest):
yield obj
- def _process_test_manifest(self, context, info, manifest_path):
+ def _process_test_manifest(self, context, info, manifest_path, mpmanifest):
flavor, install_root, install_subdir, package_tests = info
- manifest_path = mozpath.normpath(manifest_path)
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
manifest_dir = mozpath.dirname(path)
manifest_reldir = mozpath.dirname(mozpath.relpath(path,
context.config.topsrcdir))
install_prefix = mozpath.join(install_root, install_subdir)
try:
- m = manifestparser.TestManifest(manifests=[path], strict=True,
- rootdir=context.config.topsrcdir)
- defaults = m.manifest_defaults[os.path.normpath(path)]
- if not m.tests:
+ defaults = mpmanifest.manifest_defaults[os.path.normpath(path)]
+ if not mpmanifest.tests:
raise SandboxValidationError('Empty test manifest: %s'
% path, context)
- obj = TestManifest(context, path, m, flavor=flavor,
+ obj = TestManifest(context, path, mpmanifest, flavor=flavor,
install_prefix=install_prefix,
relpath=mozpath.join(manifest_reldir, mozpath.basename(path)),
dupe_manifest='dupe-manifest' in defaults)
- filtered = m.tests
+ filtered = mpmanifest.tests
# Jetpack add-on tests are expected to be generated during the
# build process so they won't exist here.
if flavor != 'jetpack-addon':
missing = [t['name'] for t in filtered if not os.path.exists(t['path'])]
if missing:
raise SandboxValidationError('Test manifest (%s) lists '
'test that does not exist: %s' % (
@@ -1185,17 +1179,17 @@ class TreeMetadataEmitter(LoggingMixin):
mozpath.dirname(test['manifest']))
obj.installs[mozpath.normpath(test['path'])] = \
((mozpath.join(out_dir, manifest_relpath)), True)
process_support_files(test)
# We also copy manifests into the output directory,
# including manifests from [include:foo] directives.
- for mpath in m.manifests():
+ for mpath in mpmanifest.manifests():
mpath = mozpath.normpath(mpath)
out_path = mozpath.join(out_dir, mozpath.basename(mpath))
obj.installs[mpath] = (out_path, False)
# Some manifests reference files that are auto generated as
# part of the build or shouldn't be installed for some
# reason. Here, we prune those files from the install set.
# FUTURE we should be able to detect autogenerated files from
@@ -1213,26 +1207,22 @@ class TreeMetadataEmitter(LoggingMixin):
yield obj
except (AssertionError, Exception):
raise SandboxValidationError('Error processing test '
'manifest file %s: %s' % (path,
'\n'.join(traceback.format_exception(*sys.exc_info()))),
context)
- def _process_reftest_manifest(self, context, flavor, manifest_path):
- manifest_path = mozpath.normpath(manifest_path)
+ def _process_reftest_manifest(self, context, flavor, manifest_path, manifest):
manifest_full_path = mozpath.normpath(mozpath.join(
context.srcdir, manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
context.config.topsrcdir))
- manifest = reftest.ReftestManifest()
- manifest.load(manifest_full_path)
-
# reftest manifests don't come from manifest parser. But they are
# similar enough that we can use the same emitted objects. Note
# that we don't perform any installs for reftests.
obj = TestManifest(context, manifest_full_path, manifest,
flavor=flavor, install_prefix='%s/' % flavor,
relpath=mozpath.join(manifest_reldir,
mozpath.basename(manifest_path)))
@@ -1245,47 +1235,24 @@ class TreeMetadataEmitter(LoggingMixin):
'head': '',
'tail': '',
'support-files': '',
'subsuite': '',
})
yield obj
- def _load_web_platform_tests_manifest(self, context, manifest_path, tests_root):
- old_path = sys.path[:]
- try:
- # Setup sys.path to include all the dependencies required to import
- # the web-platform-tests manifest parser. web-platform-tests provides
- # a the localpaths.py to do the path manipulation, which we load,
- # providing the __file__ variable so it can resolve the relative
- # paths correctly.
- paths_file = os.path.join(context.config.topsrcdir, "testing",
- "web-platform", "tests", "tools", "localpaths.py")
- _globals = {"__file__": paths_file}
- execfile(paths_file, _globals)
- import manifest as wptmanifest
- finally:
- sys.path = old_path
-
- return wptmanifest.manifest.load(tests_root, manifest_path)
-
- def _process_web_platform_tests_manifest(self, context, paths):
+ def _process_web_platform_tests_manifest(self, context, paths, manifest):
manifest_path, tests_root = paths
-
- manifest_path = mozpath.normpath(manifest_path)
manifest_full_path = mozpath.normpath(mozpath.join(
context.srcdir, manifest_path))
manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
context.config.topsrcdir))
-
tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root))
- manifest = self._load_web_platform_tests_manifest(context, manifest_full_path, tests_root)
-
# Create a equivalent TestManifest object
obj = TestManifest(context, manifest_full_path, manifest,
flavor="web-platform-tests",
relpath=mozpath.join(manifest_reldir,
mozpath.basename(manifest_path)),
install_prefix="web-platform/")
--- a/python/mozbuild/mozbuild/frontend/reader.py
+++ b/python/mozbuild/mozbuild/frontend/reader.py
@@ -1114,17 +1114,17 @@ class BuildReader(object):
if mozpath.dirname(relpath) == 'js/src' and \
not config.substs.get('JS_STANDALONE'):
config = ConfigEnvironment.from_config_status(
mozpath.join(topobjdir, reldir, 'config.status'))
config.topobjdir = topobjdir
config.external_source_dir = None
- context = Context(VARIABLES, config)
+ context = Context(VARIABLES, config, self._finder)
sandbox = MozbuildSandbox(context, metadata=metadata,
finder=self._finder)
sandbox.exec_file(path)
self._execution_time += time.time() - time_start
self._file_count += len(context.all_paths)
# Yield main context before doing any processing. This gives immediate
# consumers an opportunity to change state before our remaining
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -358,17 +358,17 @@ class TestEmitterBasic(unittest.TestCase
self.assertEqual(objs[0].program, 'test_program.prog')
self.assertEqual(objs[1].program, 'test_program1.prog')
self.assertEqual(objs[2].program, 'test_program2.prog')
def test_test_manifest_missing_manifest(self):
"""A missing manifest file should result in an error."""
reader = self.reader('test-manifest-missing-manifest')
- with self.assertRaisesRegexp(SandboxValidationError, 'IOError: Missing files'):
+ with self.assertRaisesRegexp(BuildReaderError, 'IOError: Missing files'):
self.read_topsrcdir(reader)
def test_empty_test_manifest_rejected(self):
"""A test manifest without any entries is rejected."""
reader = self.reader('test-manifest-empty')
with self.assertRaisesRegexp(SandboxValidationError, 'Empty test manifest'):
self.read_topsrcdir(reader)
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -1,23 +1,26 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
import json
import os
+import sys
import mozpack.path as mozpath
from .base import MozbuildObject
from .util import OrderedDefaultDict
from collections import defaultdict
+import manifestparser
+import reftest
def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
"""Rewrite paths in a test to be under a new base path.
This is useful for running tests from a separate location from where they
were defined.
honor_install_to_subdir and the underlying install-to-subdir field are a
@@ -283,8 +286,41 @@ REFTEST_FLAVORS = ('crashtest', 'reftest
# Web platform tests have their own manifest format and are processed separately.
WEB_PATFORM_TESTS_FLAVORS = ('web-platform-tests',)
def all_test_flavors():
return ([v[0] for v in TEST_MANIFESTS.values()] +
list(REFTEST_FLAVORS) +
list(WEB_PATFORM_TESTS_FLAVORS))
+
+# Convenience methods for test manifest reading.
+def read_manifestparser_manifest(context, manifest_path):
+ path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
+ return manifestparser.TestManifest(manifests=[path], strict=True,
+ rootdir=context.config.topsrcdir,
+ finder=context._finder)
+
+def read_reftest_manifest(context, manifest_path):
+ path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
+ manifest = reftest.ReftestManifest(finder=context._finder)
+ manifest.load(path)
+ return manifest
+
+def read_wpt_manifest(context, paths):
+ manifest_path, tests_root = paths
+ full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
+ old_path = sys.path[:]
+ try:
+ # Setup sys.path to include all the dependencies required to import
+ # the web-platform-tests manifest parser. web-platform-tests provides
+ # a the localpaths.py to do the path manipulation, which we load,
+ # providing the __file__ variable so it can resolve the relative
+ # paths correctly.
+ paths_file = os.path.join(context.config.topsrcdir, "testing",
+ "web-platform", "tests", "tools", "localpaths.py")
+ _globals = {"__file__": paths_file}
+ execfile(paths_file, _globals)
+ import manifest as wptmanifest
+ finally:
+ sys.path = old_path
+ f = context._finder.get(full_path)
+ return wptmanifest.manifest.load(tests_root, f)