Bug 1437182 - Note object files associated with linkables in the emitter. draft
authorChris Manchester <cmanchester@mozilla.com>
Mon, 12 Feb 2018 18:41:43 -0800
changeset 754258 9520b7a0f012a85c0d8196fd0ba03c96cb35f891
parent 754160 6d8f470b2579e7570f14e3db557264dc075dd654
child 754259 2e90ecd73aec6efb4cf6b6ce058d49486014f35d
push id98807
push userbmo:cmanchester@mozilla.com
push dateTue, 13 Feb 2018 02:41:51 +0000
bugs1437182
milestone60.0a1
Bug 1437182 - Note object files associated with linkables in the emitter. MozReview-Commit-ID: 3IR8TolZpKs
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/program/moz.build
python/mozbuild/mozbuild/test/frontend/data/program/test_program1.cpp
python/mozbuild/mozbuild/test/frontend/data/program/test_program2.cpp
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -360,24 +360,26 @@ class LinkageMultipleRustLibrariesError(
 
 class Linkable(ContextDerived):
     """Generic context derived container object for programs and libraries"""
     __slots__ = (
         'cxx_link',
         'lib_defines',
         'linked_libraries',
         'linked_system_libs',
+        'sources',
     )
 
     def __init__(self, context):
         ContextDerived.__init__(self, context)
         self.cxx_link = False
         self.linked_libraries = []
         self.linked_system_libs = []
         self.lib_defines = Defines(context, {})
+        self.sources = defaultdict(list)
 
     def link_library(self, obj):
         assert isinstance(obj, BaseLibrary)
         if obj.KIND != self.KIND:
             raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND))
         # Linking multiple Rust libraries into an object would result in
         # multiple copies of the Rust standard library, as well as linking
         # errors from duplicate symbols.
@@ -399,16 +401,36 @@ class Linkable(ContextDerived):
             else:
                 lib = '%s%s%s' % (
                     self.config.import_prefix,
                     lib,
                     self.config.import_suffix,
                 )
         self.linked_system_libs.append(lib)
 
+    def source_files(self):
+        all_sources = []
+        # This is ordered for reproducibility and consistently w/
+        # config/rules.mk
+        for suffix in ('.c', '.S', '.cpp', '.m', '.mm', '.s'):
+            all_sources += self.sources.get(suffix, [])
+        return all_sources
+
+    @property
+    def objs(self):
+        obj_prefix = ''
+        if self.KIND == 'host':
+            obj_prefix = 'host_'
+
+        return [mozpath.join(self.objdir, '%s%s.%s' % (obj_prefix,
+                                                       mozpath.splitext(mozpath.basename(f))[0],
+                                                       self.config.substs.get('OBJ_SUFFIX', '')))
+                for f in self.source_files()]
+
+
 class BaseProgram(Linkable):
     """Context derived container object for programs, which is a unicode
     string.
 
     This class handles automatically appending a binary suffix to the program
     name.
     If the suffix is not defined, the program name is unchanged.
     Otherwise, if the program name ends with the given suffix, it is unchanged
@@ -448,16 +470,23 @@ class HostProgram(HostMixin, BaseProgram
     KIND = 'host'
 
 
 class SimpleProgram(BaseProgram):
     """Context derived container object for each program in SIMPLE_PROGRAMS"""
     SUFFIX_VAR = 'BIN_SUFFIX'
     KIND = 'target'
 
+    def source_files(self):
+        for srcs in self.sources.values():
+            for f in srcs:
+                if mozpath.basename(mozpath.splitext(f)[0]) == mozpath.splitext(self.program)[0]:
+                    return [f]
+        return []
+
 
 class HostSimpleProgram(HostMixin, BaseProgram):
     """Context derived container object for each program in
     HOST_SIMPLE_PROGRAMS"""
     SUFFIX_VAR = 'HOST_BIN_SUFFIX'
     KIND = 'host'
 
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -942,16 +942,19 @@ class TreeMetadataEmitter(LoggingMixin):
             HOST_SOURCES=(HostSources, None, ['.c', '.mm', '.cpp']),
             UNIFIED_SOURCES=(UnifiedSources, None, ['.c', '.mm', '.cpp']),
         )
         # Track whether there are any C++ source files.
         # Technically this won't do the right thing for SIMPLE_PROGRAMS in
         # a directory with mixed C and C++ source, but it's not that important.
         cxx_sources = defaultdict(bool)
 
+        # Source files to track for linkables associated with this context.
+        ctxt_sources = defaultdict(lambda: defaultdict(list))
+
         for variable, (klass, gen_klass, suffixes) in varmap.items():
             allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes])
 
             # First ensure that we haven't been given filetypes that we don't
             # recognize.
             for f in itertools.chain(sources[variable], gen_sources[variable]):
                 ext = mozpath.splitext(f)[1]
                 if ext not in allowed_suffixes:
@@ -967,18 +970,31 @@ class TreeMetadataEmitter(LoggingMixin):
                     if canonical_suffix in ('.cpp', '.mm'):
                         cxx_sources[variable] = True
                     elif canonical_suffix in ('.s', '.S'):
                         self._asm_compile_dirs.add(context.objdir)
                     arglist = [context, list(files), canonical_suffix]
                     if variable.startswith('UNIFIED_'):
                         arglist.append(context.get('FILES_PER_UNIFIED_FILE', 16))
                     obj = cls(*arglist)
+                    srcs = obj.files
+                    if isinstance(obj, UnifiedSources) and obj.have_unified_mapping:
+                        srcs = dict(obj.unified_source_mapping).keys()
+                    ctxt_sources[variable][canonical_suffix] += sorted(srcs)
                     yield obj
 
+        if ctxt_sources:
+            for linkable in linkables:
+                for target_var in ('SOURCES', 'UNIFIED_SOURCES'):
+                    for suffix, srcs in ctxt_sources[target_var].items():
+                        linkable.sources[suffix] += srcs
+            for host_linkable in host_linkables:
+                for suffix, srcs in ctxt_sources['HOST_SOURCES'].items():
+                    host_linkable.sources[suffix] += srcs
+
         for f, flags in all_flags.iteritems():
             if flags.flags:
                 ext = mozpath.splitext(f)[1]
                 yield PerSourceFlag(context, f, flags.flags)
 
         # If there are any C++ sources, set all the linkables defined here
         # to require the C++ linker.
         for vars, linkable_items in ((('SOURCES', 'UNIFIED_SOURCES'), linkables),
--- a/python/mozbuild/mozbuild/test/frontend/data/program/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/program/moz.build
@@ -2,14 +2,15 @@
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 @template
 def Program(name):
     PROGRAM = name
 
 
 @template
-def SimplePrograms(names):
+def SimplePrograms(names, ext='.cpp'):
     SIMPLE_PROGRAMS += names
+    SOURCES += ['%s%s' % (name, ext) for name in names]
 
 Program('test_program')
 
 SimplePrograms([ 'test_program1', 'test_program2' ])
new file mode 100644
new file mode 100644
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -76,16 +76,17 @@ class TestEmitterBasic(unittest.TestCase
         substs = dict(
             ENABLE_TESTS='1' if enable_tests else '',
             BIN_SUFFIX='.prog',
             OS_TARGET='WINNT',
             COMPILE_ENVIRONMENT='1',
             STL_FLAGS=['-I/path/to/topobjdir/dist/stl_wrappers'],
             VISIBILITY_FLAGS=['-include',
                               '$(topsrcdir)/config/gcc_hidden.h'],
+            OBJ_SUFFIX='obj',
         )
         if extra_substs:
             substs.update(extra_substs)
         config = MockConfig(mozpath.join(data_path, name), extra_substs=substs)
 
         return BuildReader(config)
 
     def read_topsrcdir(self, reader, filter_common=True):
@@ -637,26 +638,36 @@ class TestEmitterBasic(unittest.TestCase
         with self.assertRaisesRegexp(SandboxValidationError,
             'Cannot install files to the root of TEST_HARNESS_FILES'):
             self.read_topsrcdir(reader)
 
     def test_program(self):
         reader = self.reader('program')
         objs = self.read_topsrcdir(reader)
 
-        self.assertEqual(len(objs), 5)
-        self.assertIsInstance(objs[0], ComputedFlags)
+        self.assertEqual(len(objs), 6)
+        self.assertIsInstance(objs[0], Sources)
         self.assertIsInstance(objs[1], ComputedFlags)
-        self.assertIsInstance(objs[2], Program)
-        self.assertIsInstance(objs[3], SimpleProgram)
+        self.assertIsInstance(objs[2], ComputedFlags)
+        self.assertIsInstance(objs[3], Program)
         self.assertIsInstance(objs[4], SimpleProgram)
+        self.assertIsInstance(objs[5], SimpleProgram)
+
+        self.assertEqual(objs[3].program, 'test_program.prog')
+        self.assertEqual(objs[4].program, 'test_program1.prog')
+        self.assertEqual(objs[5].program, 'test_program2.prog')
 
-        self.assertEqual(objs[2].program, 'test_program.prog')
-        self.assertEqual(objs[3].program, 'test_program1.prog')
-        self.assertEqual(objs[4].program, 'test_program2.prog')
+        self.assertEqual(objs[4].objs,
+                         [mozpath.join(reader.config.topobjdir,
+                                       'test_program1.%s' %
+                                       reader.config.substs['OBJ_SUFFIX'])])
+        self.assertEqual(objs[5].objs,
+                         [mozpath.join(reader.config.topobjdir,
+                                       'test_program2.%s' %
+                                       reader.config.substs['OBJ_SUFFIX'])])
 
     def test_test_manifest_missing_manifest(self):
         """A missing manifest file should result in an error."""
         reader = self.reader('test-manifest-missing-manifest')
 
         with self.assertRaisesRegexp(BuildReaderError, 'IOError: Missing files'):
             self.read_topsrcdir(reader)
 
@@ -1127,16 +1138,22 @@ class TestEmitterBasic(unittest.TestCase
             '.s': ['h.s', 'i.asm'],
         }
         for suffix, files in expected.items():
             sources = suffix_map[suffix]
             self.assertEqual(
                 sources.files,
                 [mozpath.join(reader.config.topsrcdir, f) for f in files])
 
+            for f in files:
+                self.assertIn(mozpath.join(reader.config.topobjdir,
+                                           '%s.%s' % (mozpath.splitext(f)[0],
+                                                      reader.config.substs['OBJ_SUFFIX'])),
+                              linkable.objs)
+
     def test_sources_just_c(self):
         """Test that a linkable with no C++ sources doesn't have cxx_link set."""
         reader = self.reader('sources-just-c')
         objs = self.read_topsrcdir(reader)
 
         as_flags = objs.pop()
         self.assertIsInstance(as_flags, ComputedFlags)
         flags = objs.pop()
@@ -1190,16 +1207,22 @@ class TestEmitterBasic(unittest.TestCase
             '.s': ['h.s', 'i.asm'],
         }
         for suffix, files in expected.items():
             sources = suffix_map[suffix]
             self.assertEqual(
                 sources.files,
                 [mozpath.join(reader.config.topobjdir, f) for f in files])
 
+            for f in files:
+                self.assertIn(mozpath.join(reader.config.topobjdir,
+                                           '%s.%s' % (mozpath.splitext(f)[0],
+                                                      reader.config.substs['OBJ_SUFFIX'])),
+                              linkable.objs)
+
     def test_host_sources(self):
         """Test that HOST_SOURCES works properly."""
         reader = self.reader('host-sources')
         objs = self.read_topsrcdir(reader)
 
         # This objdir will generate target flags.
         flags = objs.pop()
         self.assertIsInstance(flags, ComputedFlags)
@@ -1225,23 +1248,31 @@ class TestEmitterBasic(unittest.TestCase
             '.mm': ['e.mm', 'f.mm'],
         }
         for suffix, files in expected.items():
             sources = suffix_map[suffix]
             self.assertEqual(
                 sources.files,
                 [mozpath.join(reader.config.topsrcdir, f) for f in files])
 
+            for f in files:
+                self.assertIn(mozpath.join(reader.config.topobjdir,
+                                           'host_%s.%s' % (mozpath.splitext(f)[0],
+                                                           reader.config.substs['OBJ_SUFFIX'])),
+                              linkable.objs)
+
+
     def test_unified_sources(self):
         """Test that UNIFIED_SOURCES works properly."""
         reader = self.reader('unified-sources')
         objs = self.read_topsrcdir(reader)
 
-        # The last object is a Linkable, the second to last ComputedFlags,
+        # The last object is a ComputedFlags, the second to last a Linkable,
         # followed by ldflags, ignore them.
+        linkable = objs[-2]
         objs = objs[:-3]
         self.assertEqual(len(objs), 3)
         for o in objs:
             self.assertIsInstance(o, UnifiedSources)
 
         suffix_map = {obj.canonical_suffix: obj for obj in objs}
         self.assertEqual(len(suffix_map), 3)
 
@@ -1252,16 +1283,23 @@ class TestEmitterBasic(unittest.TestCase
         }
         for suffix, files in expected.items():
             sources = suffix_map[suffix]
             self.assertEqual(
                 sources.files,
                 [mozpath.join(reader.config.topsrcdir, f) for f in files])
             self.assertTrue(sources.have_unified_mapping)
 
+            for f in dict(sources.unified_source_mapping).keys():
+                self.assertIn(mozpath.join(reader.config.topobjdir,
+                                           '%s.%s' % (mozpath.splitext(f)[0],
+                                                      reader.config.substs['OBJ_SUFFIX'])),
+                              linkable.objs)
+
+
     def test_unified_sources_non_unified(self):
         """Test that UNIFIED_SOURCES with FILES_PER_UNIFIED_FILE=1 works properly."""
         reader = self.reader('unified-sources-non-unified')
         objs = self.read_topsrcdir(reader)
 
         # The last object is a Linkable, the second to last ComputedFlags,
         # followed by ldflags, ignore them.
         objs = objs[:-3]