bug 1384568 - Write a generated-sources.json. r?gps draft
authorTed Mielczarek <ted@mielczarek.org>
Wed, 26 Jul 2017 07:34:38 -0400
changeset 615903 a1bb9f37c0849586fa8ff8d2d691b56343a42807
parent 615779 e8400551c2e39f24c75a009ebed496c7acd7bf47
child 639320 c380dacf54de019e0e50245cf4242081c2021a48
push id70524
push userbmo:ted@mielczarek.org
push dateWed, 26 Jul 2017 14:42:35 +0000
reviewersgps
bugs1384568
milestone56.0a1
bug 1384568 - Write a generated-sources.json. r?gps Capture the list of generated source files derived from moz.build data and save it in a generated-sources.json in the objdir so that we can upload generated source files for use in crash reports and when debugging release builds. MozReview-Commit-ID: FrHcyFo0rBF
python/mozbuild/mozbuild/backend/common.py
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -9,29 +9,32 @@ import json
 import os
 
 import mozpack.path as mozpath
 
 from mozbuild.backend.base import BuildBackend
 
 from mozbuild.frontend.context import (
     Context,
+    ObjDirPath,
     Path,
     RenamedSourcePath,
     VARIABLES,
 )
 from mozbuild.frontend.data import (
     BaseProgram,
     ChromeManifestEntry,
     ConfigFileSubstitution,
     ExampleWebIDLInterface,
+    Exports,
     IPDLFile,
     FinalTargetPreprocessedFiles,
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
+    GeneratedSources,
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     SharedLibrary,
     TestWebIDLFile,
     UnifiedSources,
     XPIDLFile,
     WebIDLFile,
@@ -174,16 +177,17 @@ class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
 
     def _init(self):
         self._idl_manager = XPIDLManager(self.environment)
         self._webidls = WebIDLCollection()
         self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
+        self._generated_sources = []
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
 
         if isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
             # artifact builds.
             self._idl_manager.register_idl(obj)
@@ -272,24 +276,35 @@ class CommonBackend(BuildBackend):
         elif isinstance(obj, BaseProgram):
             self._binaries.programs.append(obj)
             return False
 
         elif isinstance(obj, SharedLibrary):
             self._binaries.shared_libraries.append(obj)
             return False
 
+        elif isinstance(obj, GeneratedSources):
+            self._handle_generated_sources(obj.files)
+            return False
+
+        elif isinstance(obj, Exports):
+            objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)]
+            if objdir_files:
+                self._handle_generated_sources(objdir_files)
+            return False
+
         else:
             return False
 
         return True
 
     def consume_finished(self):
         if len(self._idl_manager.idls):
             self._handle_idl_manager(self._idl_manager)
+            self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values())
 
         self._handle_webidl_collection(self._webidls)
 
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
 
         def files_from(ipdl):
             base = mozpath.basename(ipdl)
             root, ext = mozpath.splitext(base)
@@ -300,16 +315,17 @@ class CommonBackend(BuildBackend):
                 # .ipdl also becomes Child/Parent.cpp files
                 files.extend(['%sChild.cpp' % root,
                               '%sParent.cpp' % root])
             return files
 
         ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
 
         ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
+        self._handle_generated_sources(mozpath.relpath(mozpath.join(ipdl_dir, f), self.environment.topobjdir) for f in ipdl_cppsrcs)
         unified_source_mapping = list(group_unified_files(ipdl_cppsrcs,
                                                           unified_prefix='UnifiedProtocols',
                                                           unified_suffix='cpp',
                                                           files_per_unified_file=16))
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
         self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
 
@@ -320,16 +336,26 @@ class CommonBackend(BuildBackend):
         topobjdir = self.environment.topobjdir
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {
                 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
                 'programs': [p.to_dict() for p in self._binaries.programs],
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
+        # Write out a file listing generated sources.
+        with self._write_file(mozpath.join(topobjdir, 'generated-sources.json')) as fh:
+            d = {
+                'sources': sorted(self._generated_sources),
+            }
+            json.dump(d, fh, sort_keys=True, indent=4)
+
+    def _handle_generated_sources(self, files):
+        self._generated_sources.extend(mozpath.relpath(f, self.environment.topobjdir) for f in files)
+
     def _handle_webidl_collection(self, webidls):
         if not webidls.all_stems():
             return
 
         bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings')
 
         all_inputs = set(webidls.all_static_sources())
         for s in webidls.all_non_static_basenames():
@@ -353,17 +379,17 @@ class CommonBackend(BuildBackend):
 
         import mozwebidlcodegen
 
         manager = mozwebidlcodegen.create_build_system_manager(
             self.environment.topsrcdir,
             self.environment.topobjdir,
             mozpath.join(self.environment.topobjdir, 'dist')
         )
-
+        self._handle_generated_sources(manager.expected_build_output_files())
         # Bindings are compiled in unified mode to speed up compilation and
         # to reduce linker memory size. Note that test bindings are separated
         # from regular ones so tests bindings aren't shipped.
         unified_source_mapping = list(group_unified_files(webidls.all_regular_cpp_basenames(),
                                                           unified_prefix='UnifiedBindings',
                                                           unified_suffix='cpp',
                                                           files_per_unified_file=32))
         self._write_unified_files(unified_source_mapping, bindings_dir,