--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -1,15 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
-import itertools
import json
import os
import mozpack.path as mozpath
from mozbuild.backend.base import BuildBackend
from mozbuild.frontend.context import (
@@ -19,22 +18,21 @@ from mozbuild.frontend.context import (
RenamedSourcePath,
VARIABLES,
)
from mozbuild.frontend.data import (
BaseProgram,
ChromeManifestEntry,
ConfigFileSubstitution,
Exports,
- IPDLFile,
FinalTargetPreprocessedFiles,
FinalTargetFiles,
GeneratedSources,
GnProjectData,
- PreprocessedIPDLFile,
+ IPDLCollection,
SharedLibrary,
UnifiedSources,
XPIDLFile,
WebIDLCollection,
)
from mozbuild.jar import (
DeprecatedJarManifest,
JarManifestParser,
@@ -89,40 +87,23 @@ class XPIDLManager(object):
class BinariesCollection(object):
"""Tracks state of binaries produced by the build."""
def __init__(self):
self.shared_libraries = []
self.programs = []
-class IPDLCollection(object):
- """Collects IPDL files during the build."""
-
- def __init__(self):
- self.sources = set()
- self.preprocessed_sources = set()
-
- def all_sources(self):
- return self.sources | self.preprocessed_sources
-
- def all_regular_sources(self):
- return self.sources
-
- def all_preprocessed_sources(self):
- return self.preprocessed_sources
-
class CommonBackend(BuildBackend):
"""Holds logic common to all build backends."""
def _init(self):
self._idl_manager = XPIDLManager(self.environment)
self._binaries = BinariesCollection()
self._configs = set()
- self._ipdls = IPDLCollection()
self._generated_sources = set()
def consume_object(self, obj):
self._configs.add(obj.config)
if isinstance(obj, XPIDLFile):
# TODO bug 1240134 tracks not processing XPIDL files during
# artifact builds.
@@ -135,29 +116,26 @@ class CommonBackend(BuildBackend):
return False
with self._get_preprocessor(obj) as pp:
pp.do_include(obj.input_path)
self.backend_input_files.add(obj.input_path)
elif isinstance(obj, WebIDLCollection):
self._handle_webidl_collection(obj)
- elif isinstance(obj, PreprocessedIPDLFile):
- if self.environment.is_artifact_build:
- return True
-
- self._ipdls.preprocessed_sources.add(mozpath.join(
- obj.srcdir, obj.basename))
-
- elif isinstance(obj, IPDLFile):
- # IPDL isn't relevant to artifact builds.
- if self.environment.is_artifact_build:
- return True
-
- self._ipdls.sources.add(mozpath.join(obj.srcdir, obj.basename))
+ elif isinstance(obj, IPDLCollection):
+ self._handle_generated_sources(mozpath.join(obj.objdir, f)
+ for f in obj.all_generated_sources())
+ self._write_unified_files(obj.unified_source_mapping, obj.objdir,
+ poison_windows_h=False)
+ self._handle_ipdl_sources(obj.objdir,
+ list(sorted(obj.all_sources())),
+ list(sorted(obj.all_preprocessed_sources())),
+ list(sorted(obj.all_regular_sources())),
+ obj.unified_source_mapping)
elif isinstance(obj, UnifiedSources):
# Unified sources aren't relevant to artifact builds.
if self.environment.is_artifact_build:
return True
if obj.have_unified_mapping:
self._write_unified_files(obj.unified_source_mapping, obj.objdir)
@@ -192,44 +170,16 @@ class CommonBackend(BuildBackend):
return True
def consume_finished(self):
if len(self._idl_manager.idls):
self._handle_idl_manager(self._idl_manager)
self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values())
- sorted_ipdl_sources = list(sorted(self._ipdls.all_sources()))
- sorted_nonstatic_ipdl_sources = list(sorted(self._ipdls.all_preprocessed_sources()))
- sorted_static_ipdl_sources = list(sorted(self._ipdls.all_regular_sources()))
-
- def files_from(ipdl):
- base = mozpath.basename(ipdl)
- root, ext = mozpath.splitext(base)
-
- # Both .ipdl and .ipdlh become .cpp files
- files = ['%s.cpp' % root]
- if ext == '.ipdl':
- # .ipdl also becomes Child/Parent.cpp files
- files.extend(['%sChild.cpp' % root,
- '%sParent.cpp' % root])
- return files
-
- ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
-
- ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
- self._handle_generated_sources(mozpath.join(ipdl_dir, f) for f in ipdl_cppsrcs)
- unified_source_mapping = list(group_unified_files(ipdl_cppsrcs,
- unified_prefix='UnifiedProtocols',
- unified_suffix='cpp',
- files_per_unified_file=16))
-
- self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
- self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
- sorted_static_ipdl_sources, unified_source_mapping)
for config in self._configs:
self.backend_input_files.add(config.source)
# Write out a machine-readable file describing binaries.
topobjdir = self.environment.topobjdir
with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
d = {
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -19,16 +19,17 @@ from __future__ import absolute_import,
from mozbuild.util import StrictOrderingOnAppendList
from mozpack.chrome.manifest import ManifestEntry
import mozpack.path as mozpath
from .context import FinalTargetValue
from collections import defaultdict, OrderedDict
+import itertools
from ..util import (
group_unified_files,
)
from ..testing import (
all_test_flavors,
)
@@ -301,39 +302,57 @@ class WebIDLCollection(ContextDerived):
return [mozpath.splitext(b)[0] for b in self.all_basenames()]
def generated_events_basenames(self):
return [mozpath.basename(s) for s in self.generated_events_sources]
def generated_events_stems(self):
return [mozpath.splitext(b)[0] for b in self.generated_events_basenames()]
-class IPDLFile(ContextDerived):
- """Describes an individual .ipdl source file."""
+
+class IPDLCollection(ContextDerived):
+ """Collects IPDL files during the build."""
+
+ def __init__(self, context):
+ ContextDerived.__init__(self, context)
+ self.sources = set()
+ self.preprocessed_sources = set()
- __slots__ = (
- 'basename',
- )
+ def all_sources(self):
+ return self.sources | self.preprocessed_sources
+
+ def all_regular_sources(self):
+ return self.sources
- def __init__(self, context, path):
- ContextDerived.__init__(self, context)
+ def all_preprocessed_sources(self):
+ return self.preprocessed_sources
- self.basename = path
+ def all_generated_sources(self):
+ sorted_ipdl_sources = list(sorted(self.all_sources()))
-class PreprocessedIPDLFile(ContextDerived):
- """Describes an individual .ipdl source file that requires preprocessing."""
+ def files_from(ipdl):
+ base = mozpath.basename(ipdl)
+ root, ext = mozpath.splitext(base)
- __slots__ = (
- 'basename',
- )
+ # Both .ipdl and .ipdlh become .cpp files
+ files = ['%s.cpp' % root]
+ if ext == '.ipdl':
+ # .ipdl also becomes Child/Parent.cpp files
+ files.extend(['%sChild.cpp' % root,
+ '%sParent.cpp' % root])
+ return files
- def __init__(self, context, path):
- ContextDerived.__init__(self, context)
+ return list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
- self.basename = path
+ @property
+ def unified_source_mapping(self):
+ return list(group_unified_files(self.all_generated_sources(),
+ unified_prefix='UnifiedProtocols',
+ unified_suffix='cpp',
+ files_per_unified_file=16))
class LinkageWrongKindError(Exception):
"""Error thrown when trying to link objects of the wrong kind"""
class LinkageMultipleRustLibrariesError(Exception):
"""Error thrown when trying to link multiple Rust libraries to an object"""
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -45,27 +45,26 @@ from .data import (
ExternalSharedLibrary,
HostDefines,
HostLibrary,
HostProgram,
HostRustProgram,
HostSimpleProgram,
HostSources,
InstallationTarget,
- IPDLFile,
+ IPDLCollection,
JARManifest,
Library,
Linkable,
LocalInclude,
LocalizedFiles,
LocalizedPreprocessedFiles,
ObjdirFiles,
ObjdirPreprocessedFiles,
PerSourceFlag,
- PreprocessedIPDLFile,
WebIDLCollection,
Program,
RustLibrary,
HostRustLibrary,
RustProgram,
RustTest,
SharedLibrary,
SimpleProgram,
@@ -133,17 +132,17 @@ class TreeMetadataEmitter(LoggingMixin):
self._rust_compile_dirs = set()
self._asm_compile_dirs = set()
self._compile_flags = dict()
self._compile_as_flags = dict()
self._linkage = []
self._static_linking_shared = set()
self._crate_verified_local = set()
self._crate_directories = dict()
- self._webidls = defaultdict(set)
+ self._idls = defaultdict(set)
# Keep track of external paths (third party build systems), starting
# from what we run a subconfigure in. We'll eliminate some directories
# as we traverse them with moz.build (e.g. js/src).
subconfigures = os.path.join(self.config.topobjdir, 'subconfigures')
paths = []
if os.path.exists(subconfigures):
paths = open(subconfigures).read().splitlines()
@@ -284,32 +283,40 @@ class TreeMetadataEmitter(LoggingMixin):
yield flags_obj
for flags_obj in self._compile_as_flags.values():
yield flags_obj
for obj in self._binaries.values():
yield obj
- webidl_root = self.config.substs.get('WEBIDL_ROOT')
- if webidl_root:
- idlcollection = WebIDLCollection(contexts[webidl_root])
- webidl_attrs = [
- ('GENERATED_EVENTS_WEBIDL_FILES', idlcollection.generated_events_sources),
- ('GENERATED_WEBIDL_FILES', idlcollection.generated_sources),
- ('PREPROCESSED_TEST_WEBIDL_FILES', idlcollection.preprocessed_test_sources),
- ('PREPROCESSED_WEBIDL_FILES', idlcollection.preprocessed_sources),
- ('TEST_WEBIDL_FILES', idlcollection.test_sources),
- ('WEBIDL_FILES', idlcollection.sources),
- ('WEBIDL_EXAMPLE_INTERFACES', idlcollection.example_interfaces),
- ]
- for var, src_set in webidl_attrs:
- src_set |= self._webidls[var]
+ webidl_attrs = [
+ ('GENERATED_EVENTS_WEBIDL_FILES', lambda c: c.generated_events_sources),
+ ('GENERATED_WEBIDL_FILES', lambda c: c.generated_sources),
+ ('PREPROCESSED_TEST_WEBIDL_FILES', lambda c: c.preprocessed_test_sources),
+ ('PREPROCESSED_WEBIDL_FILES', lambda c: c.preprocessed_sources),
+ ('TEST_WEBIDL_FILES', lambda c: c.test_sources),
+ ('WEBIDL_FILES', lambda c: c.sources),
+ ('WEBIDL_EXAMPLE_INTERFACES', lambda c: c.example_interfaces),
+ ]
+ ipdl_attrs = [
+ ('IPDL_SOURCES', lambda c: c.sources),
+ ('PREPROCESSED_IPDL_SOURCES', lambda c: c.preprocessed_sources),
+ ]
- yield idlcollection
+ for root, cls, attrs in ((self.config.substs.get('WEBIDL_ROOT'),
+ WebIDLCollection, webidl_attrs),
+ (self.config.substs.get('IPDL_ROOT'),
+ IPDLCollection, ipdl_attrs)):
+ if root:
+ collection = cls(contexts[root])
+ for var, src_getter in attrs:
+ src_getter(collection).update(self._idls[var])
+
+ yield collection
LIBRARY_NAME_VAR = {
'host': 'HOST_LIBRARY_NAME',
'target': 'LIBRARY_NAME',
}
LIBSTDCXX_VAR = {
@@ -1112,38 +1119,32 @@ class TreeMetadataEmitter(LoggingMixin):
# initialized values if present end up in computed flags.
defines_obj = cls(context, context[defines_var])
defines_from_obj = list(defines_obj.get_defines())
if defines_from_obj:
for flags in backend_flags:
flags.resolve_flags(defines_var, defines_from_obj)
- simple_lists = [
- ('IPDL_SOURCES', IPDLFile),
- ('PREPROCESSED_IPDL_SOURCES', PreprocessedIPDLFile),
- ]
- for context_var, klass in simple_lists:
- for name in context.get(context_var, []):
- yield klass(context, name)
-
- webidl_vars = (
+ idl_vars = (
'GENERATED_EVENTS_WEBIDL_FILES',
'GENERATED_WEBIDL_FILES',
'PREPROCESSED_TEST_WEBIDL_FILES',
'PREPROCESSED_WEBIDL_FILES',
'TEST_WEBIDL_FILES',
'WEBIDL_FILES',
+ 'IPDL_SOURCES',
+ 'PREPROCESSED_IPDL_SOURCES',
)
- for context_var in webidl_vars:
+ for context_var in idl_vars:
for name in context.get(context_var, []):
- self._webidls[context_var].add(mozpath.join(context.srcdir, name))
+ self._idls[context_var].add(mozpath.join(context.srcdir, name))
# WEBIDL_EXAMPLE_INTERFACES do not correspond to files.
for name in context.get('WEBIDL_EXAMPLE_INTERFACES', []):
- self._webidls['WEBIDL_EXAMPLE_INTERFACES'].add(name)
+ self._idls['WEBIDL_EXAMPLE_INTERFACES'].add(name)
local_includes = []
for local_include in context.get('LOCAL_INCLUDES', []):
full_path = local_include.full_path
if (not isinstance(local_include, ObjDirPath) and
not os.path.exists(full_path)):
raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
'does not exist: %s (resolved to %s)' % (local_include,
--- a/python/mozbuild/mozbuild/test/backend/common.py
+++ b/python/mozbuild/mozbuild/test/backend/common.py
@@ -183,16 +183,25 @@ CONFIGS = defaultdict(lambda: {
'RUST_TARGET': 'x86_64-unknown-linux-gnu',
'LIB_PREFIX': 'lib',
'RUST_LIB_PREFIX': 'lib',
'LIB_SUFFIX': 'a',
'RUST_LIB_SUFFIX': 'a',
'OS_TARGET': 'Darwin',
},
},
+ 'ipdl_sources': {
+ 'defines': {},
+ 'non_global_defines': [],
+ 'substs': {
+ 'COMPILE_ENVIRONMENT': '1',
+ 'LIB_SUFFIX': '.a',
+ 'BIN_SUFFIX': '',
+ },
+ },
})
class BackendTester(unittest.TestCase):
def setUp(self):
self._old_env = dict(os.environ)
os.environ.pop('MOZ_OBJDIR', None)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -652,29 +652,35 @@ class TestRecursiveMakeBackend(BackendTe
full = mozpath.join(man_dir, e)
self.assertTrue(os.path.exists(full))
m2 = InstallManifest(path=full)
self.assertEqual(m, m2)
def test_ipdl_sources(self):
"""Test that PREPROCESSED_IPDL_SOURCES and IPDL_SOURCES are written to ipdlsrcs.mk correctly."""
- env = self._consume('ipdl_sources', RecursiveMakeBackend)
+ env = self._get_environment('ipdl_sources')
- manifest_path = mozpath.join(env.topobjdir,
- 'ipc', 'ipdl', 'ipdlsrcs.mk')
+ # Make substs writable so we can set the value of IPDL_ROOT to reflect
+ # the correct objdir.
+ env.substs = dict(env.substs)
+ env.substs['IPDL_ROOT'] = env.topobjdir
+
+ self._consume('ipdl_sources', RecursiveMakeBackend, env)
+
+ manifest_path = mozpath.join(env.topobjdir, 'ipdlsrcs.mk')
lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
# Handle Windows paths correctly
topsrcdir = env.topsrcdir.replace(os.sep, '/')
expected = [
"ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4),
"CPPSRCS := UnifiedProtocols0.cpp",
- "IPDLDIRS := %s/ipc/ipdl %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir),
+ "IPDLDIRS := %s %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir),
]
found = [str for str in lines if str.startswith(('ALL_IPDLSRCS',
'CPPSRCS',
'IPDLDIRS'))]
self.assertEqual(found, expected)
def test_defines(self):
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -23,23 +23,22 @@ from mozbuild.frontend.data import (
Exports,
FinalTargetPreprocessedFiles,
GeneratedFile,
GeneratedSources,
HostDefines,
HostRustLibrary,
HostRustProgram,
HostSources,
- IPDLFile,
+ IPDLCollection,
JARManifest,
LinkageMultipleRustLibrariesError,
LocalInclude,
LocalizedFiles,
LocalizedPreprocessedFiles,
- PreprocessedIPDLFile,
Program,
RustLibrary,
RustProgram,
SharedLibrary,
SimpleProgram,
Sources,
StaticLibrary,
TestHarnessFiles,
@@ -918,42 +917,60 @@ class TestEmitterBasic(unittest.TestCase
"""Missing test files should result in error, even when the test list is not filtered."""
reader = self.reader('test-manifest-missing-test-file-unfiltered')
with self.assertRaisesRegexp(SandboxValidationError,
'lists test that does not exist: missing.js'):
self.read_topsrcdir(reader)
def test_ipdl_sources(self):
- reader = self.reader('ipdl_sources')
+ reader = self.reader('ipdl_sources',
+ extra_substs={'IPDL_ROOT': mozpath.abspath('/path/to/topobjdir')})
objs = self.read_topsrcdir(reader)
+ ipdl_collection = objs[0]
+ self.assertIsInstance(ipdl_collection, IPDLCollection)
- ipdls = []
- nonstatic_ipdls = []
- for o in objs:
- if isinstance(o, IPDLFile):
- ipdls.append('%s/%s' % (o.relsrcdir, o.basename))
- elif isinstance(o, PreprocessedIPDLFile):
- nonstatic_ipdls.append('%s/%s' % (o.relsrcdir, o.basename))
-
- expected = [
+ ipdls = set(mozpath.relpath(p, ipdl_collection.topsrcdir)
+ for p in ipdl_collection.all_regular_sources())
+ expected = set([
'bar/bar.ipdl',
'bar/bar2.ipdlh',
'foo/foo.ipdl',
'foo/foo2.ipdlh',
- ]
+ ])
self.assertEqual(ipdls, expected)
- expected = [
+ pp_ipdls = set(mozpath.relpath(p, ipdl_collection.topsrcdir)
+ for p in ipdl_collection.all_preprocessed_sources())
+ expected = set([
'bar/bar1.ipdl',
'foo/foo1.ipdl',
- ]
+ ])
+ self.assertEqual(pp_ipdls, expected)
- self.assertEqual(nonstatic_ipdls, expected)
+ generated_sources = set(ipdl_collection.all_generated_sources())
+ expected = set([
+ 'bar.cpp',
+ 'barChild.cpp',
+ 'barParent.cpp',
+ 'bar1.cpp',
+ 'bar1Child.cpp',
+ 'bar1Parent.cpp',
+ 'bar2.cpp',
+ 'foo.cpp',
+ 'fooChild.cpp',
+ 'fooParent.cpp',
+ 'foo1.cpp',
+ 'foo1Child.cpp',
+ 'foo1Parent.cpp',
+ 'foo2.cpp'
+ ])
+ self.assertEqual(generated_sources, expected)
+
def test_local_includes(self):
"""Test that LOCAL_INCLUDES is emitted correctly."""
reader = self.reader('local_includes')
objs = self.read_topsrcdir(reader)
local_includes = [o.path for o in objs if isinstance(o, LocalInclude)]
expected = [