Move includes to computed flags. draft
authorChris Manchester <cmanchester@mozilla.com>
Thu, 17 Aug 2017 16:04:41 -0700
changeset 648550 971dcf364cf988563f036b47bf79330a2acb93d9
parent 648549 386477ae2a999f8cccec6040ae5bfb168b427f45
child 648551 fb0682de85ec4a032fcd7d9d025fef8fd75b2959
push id74790
push userbmo:cmanchester@mozilla.com
push dateThu, 17 Aug 2017 23:05:10 +0000
milestone57.0a1
Move includes to computed flags. MozReview-Commit-ID: Ec0wZfoV63B
config/config.mk
config/rules.mk
python/mozbuild/mozbuild/compilation/database.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -323,18 +323,18 @@ endif # CLANG_CL
 
 # Use warnings-as-errors if ALLOW_COMPILER_WARNINGS is not set to 1 (which
 # includes the case where it's undefined).
 ifneq (1,$(ALLOW_COMPILER_WARNINGS))
 CXXFLAGS += $(WARNINGS_AS_ERRORS)
 CFLAGS   += $(WARNINGS_AS_ERRORS)
 endif # ALLOW_COMPILER_WARNINGS
 
-COMPILE_CFLAGS	= $(COMPUTED_CFLAGS) $(INCLUDES) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CFLAGS) $(_DEPEND_CFLAGS) $(CFLAGS) $(MOZBUILD_CFLAGS) $(MK_COMPILE_DEFINES)
-COMPILE_CXXFLAGS = $(COMPUTED_CXXFLAGS) $(INCLUDES) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CXXFLAGS) $(_DEPEND_CFLAGS) $(CXXFLAGS) $(MOZBUILD_CXXFLAGS) $(MK_COMPILE_DEFINES)
+COMPILE_CFLAGS	= $(COMPUTED_CFLAGS) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CFLAGS) $(_DEPEND_CFLAGS) $(CFLAGS) $(MOZBUILD_CFLAGS) $(MK_COMPILE_DEFINES)
+COMPILE_CXXFLAGS = $(COMPUTED_CXXFLAGS) $(OS_INCLUDES) $(DSO_CFLAGS) $(DSO_PIC_CFLAGS) $(RTL_FLAGS) $(OS_COMPILE_CXXFLAGS) $(_DEPEND_CFLAGS) $(CXXFLAGS) $(MOZBUILD_CXXFLAGS) $(MK_COMPILE_DEFINES)
 COMPILE_CMFLAGS = $(OS_COMPILE_CMFLAGS) $(MOZBUILD_CMFLAGS)
 COMPILE_CMMFLAGS = $(OS_COMPILE_CMMFLAGS) $(MOZBUILD_CMMFLAGS)
 ASFLAGS += $(MOZBUILD_ASFLAGS)
 
 ifndef CROSS_COMPILE
 HOST_CFLAGS += $(RTL_FLAGS)
 endif
 
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -63,17 +63,16 @@ testxpcobjdir = $(DEPTH)/_tests/xpcshell
 ifdef ENABLE_TESTS
 ifdef CPP_UNIT_TESTS
 ifdef COMPILE_ENVIRONMENT
 
 # Compile the tests to $(DIST)/bin.  Make lots of niceties available by default
 # through TestHarness.h, by modifying the list of includes and the libs against
 # which stuff links.
 SIMPLE_PROGRAMS += $(CPP_UNIT_TESTS)
-INCLUDES += -I$(ABS_DIST)/include/testing
 
 ifndef MOZ_PROFILE_GENERATE
 CPP_UNIT_TESTS_FILES = $(CPP_UNIT_TESTS)
 CPP_UNIT_TESTS_DEST = $(DIST)/cppunittests
 CPP_UNIT_TESTS_TARGET = target
 INSTALL_TARGETS += CPP_UNIT_TESTS
 endif
 
--- a/python/mozbuild/mozbuild/compilation/database.py
+++ b/python/mozbuild/mozbuild/compilation/database.py
@@ -39,23 +39,19 @@ class CompileDBBackend(CommonBackend):
 
         # The database we're going to dump out to.
         self._db = OrderedDict()
 
         # The cache for per-directory flags
         self._flags = {}
 
         self._envs = {}
-        self._includes = defaultdict(list)
         self._local_flags = defaultdict(dict)
         self._per_source_flags = defaultdict(list)
-        self._extra_includes = defaultdict(list)
         self._gyp_dirs = set()
-        self._dist_include_testing = '-I%s' % mozpath.join(
-            self.environment.topobjdir, 'dist', 'include', 'testing')
 
     def consume_object(self, obj):
         # Those are difficult directories, that will be handled later.
         if obj.relativedir in (
                 'build/unix/elfhack',
                 'build/unix/elfhack/inject',
                 'build/clang-plugin',
                 'build/clang-plugin/tests',
@@ -75,27 +71,16 @@ class CompileDBBackend(CommonBackend):
                     self._local_flags[obj.objdir][var] = value
 
         elif isinstance(obj, (Sources, GeneratedSources)):
             # For other sources, include each source file.
             for f in obj.files:
                 self._build_db_line(obj.objdir, obj.relativedir, obj.config, f,
                                     obj.canonical_suffix)
 
-        elif isinstance(obj, LocalInclude):
-            self._includes[obj.objdir].append('-I%s' % mozpath.normpath(
-                obj.path.full_path))
-
-        elif isinstance(obj, Linkable):
-            if isinstance(obj, SimpleProgram) and obj.is_unit_test:
-                if (self._dist_include_testing not in
-                        self._extra_includes[obj.objdir]):
-                    self._extra_includes[obj.objdir].append(
-                        self._dist_include_testing)
-
         elif isinstance(obj, VariablePassthru):
             if obj.variables.get('IS_GYP_DIR'):
                 self._gyp_dirs.add(obj.objdir)
             for var in ('MOZBUILD_CFLAGS', 'MOZBUILD_CXXFLAGS',
                         'MOZBUILD_CMFLAGS', 'MOZBUILD_CMMFLAGS',
                         'RTL_FLAGS'):
                 if var in obj.variables:
                     self._local_flags[obj.objdir][var] = obj.variables[var]
@@ -119,32 +104,31 @@ class CompileDBBackend(CommonBackend):
 
         for (directory, filename, unified), cmd in self._db.iteritems():
             env = self._envs[directory]
             cmd = list(cmd)
             if unified is None:
                 cmd.append(filename)
             else:
                 cmd.append(unified)
-            local_extra = list(self._extra_includes[directory])
+            os_includes = []
             if directory not in self._gyp_dirs:
                 for var in (
                     'NSPR_CFLAGS',
                     'NSS_CFLAGS',
                     'MOZ_JPEG_CFLAGS',
                     'MOZ_PNG_CFLAGS',
                     'MOZ_ZLIB_CFLAGS',
                     'MOZ_PIXMAN_CFLAGS',
                 ):
                     f = env.substs.get(var)
                     if f:
-                        local_extra.extend(f)
+                        os_includes.extend(f)
             variables = {
-                'LOCAL_INCLUDES': self._includes[directory],
-                'EXTRA_INCLUDES': local_extra,
+                'OS_INCLUDES': os_includes,
                 'DIST': mozpath.join(env.topobjdir, 'dist'),
                 'DEPTH': env.topobjdir,
                 'MOZILLA_DIR': env.topsrcdir,
                 'topsrcdir': env.topsrcdir,
                 'topobjdir': env.topobjdir,
             }
             variables.update(self._local_flags[directory])
             c = []
@@ -226,21 +210,17 @@ class CompileDBBackend(CommonBackend):
                 return
             if isinstance(value, types.StringTypes):
                 value = value.split()
             db.extend(value)
 
         db.append('$(COMPUTED_%s)' % self.CFLAGS[canonical_suffix])
 
         db.extend((
-            '-I%s' % mozpath.join(cenv.topsrcdir, reldir),
-            '-I%s' % objdir,
-            '$(LOCAL_INCLUDES)',
-            '-I%s/dist/include' % cenv.topobjdir,
-            '$(EXTRA_INCLUDES)',
+            '$(OS_INCLUDES)',
         ))
         append_var('DSO_CFLAGS')
         append_var('DSO_PIC_CFLAGS')
         if canonical_suffix in ('.c', '.cpp'):
             db.append('$(RTL_FLAGS)')
         append_var('OS_COMPILE_%s' % self.CFLAGS[canonical_suffix])
         append_var('OS_CPPFLAGS')
         append_var('OS_%s' % self.CFLAGS[canonical_suffix])
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -294,22 +294,33 @@ class InitializedDefines(ContextDerivedV
         for define in context.config.substs.get('MOZ_DEBUG_DEFINES', ()):
             self[define] = 1
         if value:
             self.update(value)
 
 
 class CompileFlags(ContextDerivedValue, dict):
     def __init__(self, context):
+        self.context = context
+        # We want to refer to the "main srcdir", not the one relative to an included
+        # mozbuild file in case this object gets initiated in an included mozbuild
+        # file.
+        main_src_dir = mozpath.dirname(context.main_path)
+
         self.flag_variables = [
             ('STL', (context.config.substs.get('STL_FLAGS'), ['CXXFLAGS'])),
             ('VISIBILITY', (context.config.substs.get('VISIBILITY_FLAGS'),
                             ['CXXFLAGS', 'CFLAGS'])),
             ('DEFINES', (None, ['CXXFLAGS', 'CFLAGS'])),
             ('LIBRARY_DEFINES', (None, ['CXXFLAGS', 'CFLAGS'])),
+            ('BASE_INCLUDES', (['-I%s' % main_src_dir, '-I%s' % context.objdir],
+                               ['CXXFLAGS', 'CFLAGS'])),
+            ('LOCAL_INCLUDES', (None, ['CXXFLAGS', 'CFLAGS'])),
+            ('EXTRA_INCLUDES', (['-I%s/dist/include' % context.config.topobjdir],
+                                ['CXXFLAGS', 'CFLAGS'])),
         ]
         dict.__init__(self, [
             (k, v) for k, (v, _) in self.flag_variables if v
         ])
 
 
 class FinalTargetValue(ContextDerivedValue, unicode):
     def __new__(cls, context, value=""):
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1012,23 +1012,34 @@ class TreeMetadataEmitter(LoggingMixin):
             ('TEST_WEBIDL_FILES', TestWebIDLFile),
             ('WEBIDL_FILES', WebIDLFile),
             ('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface),
         ]
         for context_var, klass in simple_lists:
             for name in context.get(context_var, []):
                 yield klass(context, name)
 
+        local_includes = []
         for local_include in context.get('LOCAL_INCLUDES', []):
             if (not isinstance(local_include, ObjDirPath) and
                     not os.path.exists(local_include.full_path)):
                 raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
                     'does not exist: %s (resolved to %s)' % (local_include,
                     local_include.full_path), context)
-            yield LocalInclude(context, local_include)
+            include_obj = LocalInclude(context, local_include)
+            local_includes.append(include_obj.path.full_path)
+            yield include_obj
+
+        computed_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes])
+
+        if len(context['CPP_UNIT_TESTS']):
+            # TODO: Move this to the CppUnitTests template.
+            dist_include_testing = mozpath.join(self.config.topobjdir,
+                                                'dist', 'include', 'testing')
+            computed_flags.flags['EXTRA_INCLUDES'].append('-I%s' % dist_include_testing)
 
         for obj in self._handle_linkables(context, passthru, generated_files):
             yield obj
 
         generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()])
 
         components = []
         for var, cls in (
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -748,63 +748,63 @@ class TestRecursiveMakeBackend(BackendTe
         found = [str for str in lines if str.startswith('LOCAL_INCLUDES')]
         self.assertEqual(found, expected)
 
     def test_rust_library(self):
         """Test that a Rust library is written to backend.mk correctly."""
         env = self._consume('rust-library', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-2]]
 
         expected = [
             'RUST_LIBRARY_FILE := ./x86_64-unknown-linux-gnu/release/libtest_library.a',
             'CARGO_FILE := $(srcdir)/Cargo.toml',
             'CARGO_TARGET_DIR := %s' % env.topobjdir,
         ]
 
         self.assertEqual(lines, expected)
 
     def test_host_rust_library(self):
         """Test that a Rust library is written to backend.mk correctly."""
         env = self._consume('host-rust-library', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-2]]
 
         expected = [
             'HOST_RUST_LIBRARY_FILE := ./x86_64-unknown-linux-gnu/release/libhostrusttool.a',
             'CARGO_FILE := $(srcdir)/Cargo.toml',
             'CARGO_TARGET_DIR := %s' % env.topobjdir,
         ]
 
         self.assertEqual(lines, expected)
 
     def test_host_rust_library_with_features(self):
         """Test that a host Rust library with features is written to backend.mk correctly."""
         env = self._consume('host-rust-library-features', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-2]]
 
         expected = [
             'HOST_RUST_LIBRARY_FILE := ./x86_64-unknown-linux-gnu/release/libhostrusttool.a',
             'CARGO_FILE := $(srcdir)/Cargo.toml',
             'CARGO_TARGET_DIR := %s' % env.topobjdir,
             'HOST_RUST_LIBRARY_FEATURES := musthave cantlivewithout',
         ]
 
         self.assertEqual(lines, expected)
 
     def test_rust_library_with_features(self):
         """Test that a Rust library with features is written to backend.mk correctly."""
         env = self._consume('rust-library-features', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:-2]]
 
         expected = [
             'RUST_LIBRARY_FILE := ./x86_64-unknown-linux-gnu/release/libfeature_library.a',
             'CARGO_FILE := $(srcdir)/Cargo.toml',
             'CARGO_TARGET_DIR := %s' % env.topobjdir,
             'RUST_LIBRARY_FEATURES := musthave cantlivewithout',
         ]
 
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -207,16 +207,20 @@ class TestEmitterBasic(unittest.TestCase
         self.maxDiff = maxDiff
 
     def test_compile_flags(self):
         reader = self.reader('compile-flags')
         sources, lib, flags = self.read_topsrcdir(reader)
         self.assertIsInstance(flags, ComputedFlags)
         self.assertEqual(flags.flags['STL'], reader.config.substs['STL_FLAGS'])
         self.assertEqual(flags.flags['VISIBILITY'], reader.config.substs['VISIBILITY_FLAGS'])
+        self.assertEqual(flags.flags['BASE_INCLUDES'],
+                         ['-I%s' % reader.config.topsrcdir, '-I%s' % reader.config.topobjdir])
+        self.assertEqual(flags.flags['LOCAL_INCLUDES'], [])
+        self.assertEqual(flags.flags['EXTRA_INCLUDES'], ['-I%s/dist/include' % reader.config.topobjdir])
         self.assertNotIn('DEFINES', flags.flags)
         self.assertNotIn('LIBRARY_DEFINES', flags.flags)
 
     def test_disable_stl_wrapping(self):
         reader = self.reader('disable-stl-wrapping')
         sources, lib, flags = self.read_topsrcdir(reader)
         self.assertIsInstance(flags, ComputedFlags)
         self.assertEqual(flags.flags['STL'], [])