Bug 1384400 - Add input to output mappings to mozpack.FileRegistry. r=gps draft
authorNick Alexander <nalexander@mozilla.com>
Tue, 25 Jul 2017 12:28:31 -0700
changeset 616397 1ff6a1a51bc76efa78eb564cd8e572777dace0f6
parent 615935 388d81ed93fa640f91d155f36254667c734157cf
child 616491 997dd298b3f9fe509008f77302bb0aecb96554e1
push id70662
push usernalexander@mozilla.com
push dateThu, 27 Jul 2017 00:16:35 +0000
reviewersgps
bugs1384400
milestone56.0a1
Bug 1384400 - Add input to output mappings to mozpack.FileRegistry. r=gps This is necessary because the existing manifests don't expose full dependency information. I needed to avoid the existing dependency files because those code paths need to know the output destination of the manifest in order to parse the Make dependency files; trying to adapt this system is more complicated than just preprocessing each file to extract dependency information directly. MozReview-Commit-ID: 5m0SEqmhJMM
python/mozbuild/mozpack/copier.py
python/mozbuild/mozpack/files.py
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -11,16 +11,17 @@ import sys
 from mozpack.errors import errors
 from mozpack.files import (
     BaseFile,
     Dest,
 )
 import mozpack.path as mozpath
 import errno
 from collections import (
+    defaultdict,
     Counter,
     OrderedDict,
 )
 import concurrent.futures as futures
 
 
 class FileRegistry(object):
     '''
@@ -148,16 +149,44 @@ class FileRegistry(object):
         '''
         Return the set of directories required by the paths in the container,
         in no particular order.  The returned directories are relative to an
         unspecified (virtual) root directory (and do not include said root
         directory).
         '''
         return set(k for k, v in self._required_directories.items() if v > 0)
 
+    def output_to_inputs_tree(self):
+        '''
+        Return a dictionary mapping each output path to the set of its
+        required input paths.
+
+        All paths are normalized.
+        '''
+        tree = {}
+        for output, file in self:
+            output = mozpath.normpath(output)
+            tree[output] = set(mozpath.normpath(f) for f in file.inputs())
+        return tree
+
+    def input_to_outputs_tree(self):
+        '''
+        Return a dictionary mapping each input path to the set of
+        impacted output paths.
+
+        All paths are normalized.
+        '''
+        tree = defaultdict(set)
+        for output, file in self:
+            output = mozpath.normpath(output)
+            for input in file.inputs():
+                input = mozpath.normpath(input)
+                tree[input].add(output)
+        return dict(tree)
+
 
 class FileRegistrySubtree(object):
     '''A proxy class to give access to a subtree of an existing FileRegistry.
 
     Note this doesn't implement the whole FileRegistry interface.'''
     def __new__(cls, base, registry):
         if not base:
             return registry
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -229,16 +229,22 @@ class BaseFile(object):
 
     @property
     def mode(self):
         '''
         Return the file's unix mode, or None if it has no meaning.
         '''
         return None
 
+    def inputs(self):
+        '''
+        Return an iterable of the input file paths that impact this output file.
+        '''
+        raise NotImplementedError('BaseFile.inputs() not implemented.')
+
 
 class File(BaseFile):
     '''
     File class for plain files.
     '''
     def __init__(self, path):
         self.path = path
 
@@ -256,16 +262,19 @@ class File(BaseFile):
     def read(self):
         '''Return the contents of the file.'''
         with open(self.path, 'rb') as fh:
             return fh.read()
 
     def size(self):
         return os.stat(self.path).st_size
 
+    def inputs(self):
+        return (self.path,)
+
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
     def copy(self, dest, skip_if_older=True):
         real_dest = dest
@@ -469,32 +478,46 @@ class ExistingFile(BaseFile):
 
         if not self.required:
             return
 
         if not dest.exists():
             errors.fatal("Required existing file doesn't exist: %s" %
                 dest.path)
 
+    def inputs(self):
+        return ()
+
 
 class PreprocessedFile(BaseFile):
     '''
     File class for a file that is preprocessed. PreprocessedFile.copy() runs
     the preprocessor on the file to create the output.
     '''
     def __init__(self, path, depfile_path, marker, defines, extra_depends=None,
                  silence_missing_directive_warnings=False):
         self.path = path
         self.depfile = depfile_path
         self.marker = marker
         self.defines = defines
         self.extra_depends = list(extra_depends or [])
         self.silence_missing_directive_warnings = \
             silence_missing_directive_warnings
 
+    def inputs(self):
+        pp = Preprocessor(defines=self.defines, marker=self.marker)
+        pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
+
+        with open(self.path, 'rU') as input:
+            with open(os.devnull, 'w') as output:
+                pp.processFile(input=input, output=output)
+
+        # This always yields at least self.path.
+        return pp.includes
+
     def copy(self, dest, skip_if_older=True):
         '''
         Invokes the preprocessor to create the destination file.
         '''
         if isinstance(dest, basestring):
             dest = Dest(dest)
         else:
             assert isinstance(dest, Dest)
@@ -560,16 +583,19 @@ class GeneratedFile(BaseFile):
         return BytesIO(self.content)
 
     def read(self):
         return self.content
 
     def size(self):
         return len(self.content)
 
+    def inputs(self):
+        return ()
+
 
 class DeflatedFile(BaseFile):
     '''
     File class for members of a jar archive. DeflatedFile.copy() effectively
     extracts the file from the jar archive.
     '''
     def __init__(self, file):
         from mozpack.mozjar import JarFileReader