Bug 1384241 - Part 1: Add input to output mappings. r=gps draft
authorNick Alexander <nalexander@mozilla.com>
Tue, 25 Jul 2017 12:28:31 -0700
changeset 615316 ef2c4f979fe158a39a9e51a108d6f3d1e6f766d7
parent 615176 32d9d1e81cc607320a36391845917f645f7a7f72
child 615317 b14a3acfdbe301f804e1fb27f5f8f25a0ed90626
push id70309
push usernalexander@mozilla.com
push dateTue, 25 Jul 2017 19:38:56 +0000
reviewersgps
bugs1384241
milestone56.0a1
Bug 1384241 - Part 1: Add input to output mappings. r=gps This is necessary because the existing manifests don't expose full dependency information. I needed to avoid the existing dependency files because those code paths need to know the output destination of the manifest in order to parse the Make dependency files; trying to adapt this system is more complicated than just preprocessing each file to extract dependency information directly. If this patch series meets with favor, we can add tests for this functionality. MozReview-Commit-ID: 5m0SEqmhJMM
python/mozbuild/mozpack/copier.py
python/mozbuild/mozpack/files.py
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -11,16 +11,17 @@ import sys
 from mozpack.errors import errors
 from mozpack.files import (
     BaseFile,
     Dest,
 )
 import mozpack.path as mozpath
 import errno
 from collections import (
+    defaultdict,
     Counter,
     OrderedDict,
 )
 import concurrent.futures as futures
 
 
 class FileRegistry(object):
     '''
@@ -148,16 +149,44 @@ class FileRegistry(object):
         '''
         Return the set of directories required by the paths in the container,
         in no particular order.  The returned directories are relative to an
         unspecified (virtual) root directory (and do not include said root
         directory).
         '''
         return set(k for k, v in self._required_directories.items() if v > 0)
 
+    def output_to_inputs_tree(self):
+        '''
+        Return a dictionary mapping each output path to the set of its
+        required input paths.
+
+        All paths are normalized.
+        '''
+        tree = {}
+        for output, file in self:
+            output = mozpath.normpath(output)
+            tree[output] = set(mozpath.normpath(f) for f in file.inputs())
+        return tree
+
+    def input_to_outputs_tree(self):
+        '''
+        Return a dictionary mapping each input path to the set of
+        impacted output paths.
+
+        All paths are normalized.
+        '''
+        tree = defaultdict(set)
+        for output, file in self:
+            output = mozpath.normpath(output)
+            for input in file.inputs():
+                input = mozpath.normpath(input)
+                tree[input].add(output)
+        return dict(tree.iteritems())
+
 
 class FileRegistrySubtree(object):
     '''A proxy class to give access to a subtree of an existing FileRegistry.
 
     Note this doesn't implement the whole FileRegistry interface.'''
     def __new__(cls, base, registry):
         if not base:
             return registry
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -229,16 +229,22 @@ class BaseFile(object):
 
     @property
     def mode(self):
         '''
         Return the file's unix mode, or None if it has no meaning.
         '''
         return None
 
+    def inputs(self):
+        '''
+        Return an iterator of the input file paths that impact this output file.
+        '''
+        raise NotImplementedError('BaseFile.inputs() not implemented.')
+
 
 class File(BaseFile):
     '''
     File class for plain files.
     '''
     def __init__(self, path):
         self.path = path
 
@@ -256,16 +262,19 @@ class File(BaseFile):
     def read(self):
         '''Return the contents of the file.'''
         with open(self.path, 'rb') as fh:
             return fh.read()
 
     def size(self):
         return os.stat(self.path).st_size
 
+    def inputs(self):
+        yield self.path
+
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
     def copy(self, dest, skip_if_older=True):
         real_dest = dest
@@ -469,32 +478,46 @@ class ExistingFile(BaseFile):
 
         if not self.required:
             return
 
         if not dest.exists():
             errors.fatal("Required existing file doesn't exist: %s" %
                 dest.path)
 
+    def inputs(self):
+        return iter(())
+
 
 class PreprocessedFile(BaseFile):
     '''
     File class for a file that is preprocessed. PreprocessedFile.copy() runs
     the preprocessor on the file to create the output.
     '''
     def __init__(self, path, depfile_path, marker, defines, extra_depends=None,
                  silence_missing_directive_warnings=False):
         self.path = path
         self.depfile = depfile_path
         self.marker = marker
         self.defines = defines
         self.extra_depends = list(extra_depends or [])
         self.silence_missing_directive_warnings = \
             silence_missing_directive_warnings
 
+    def inputs(self):
+        pp = Preprocessor(defines=self.defines, marker=self.marker)
+        pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
+
+        with open(self.path, 'rU') as input:
+            with open(os.devnull, 'w') as output:
+                pp.processFile(input=input, output=output)
+
+        # This always yields at least self.path.
+        return iter(pp.includes)
+
     def copy(self, dest, skip_if_older=True):
         '''
         Invokes the preprocessor to create the destination file.
         '''
         if isinstance(dest, basestring):
             dest = Dest(dest)
         else:
             assert isinstance(dest, Dest)
@@ -560,16 +583,19 @@ class GeneratedFile(BaseFile):
         return BytesIO(self.content)
 
     def read(self):
         return self.content
 
     def size(self):
         return len(self.content)
 
+    def inputs(self):
+        return iter(())
+
 
 class DeflatedFile(BaseFile):
     '''
     File class for members of a jar archive. DeflatedFile.copy() effectively
     extracts the file from the jar archive.
     '''
     def __init__(self, file):
         from mozpack.mozjar import JarFileReader