Bug 1384400 - Add input to output mappings to mozpack.FileRegistry. r=gps
This is necessary because the existing manifests don't expose full
dependency information. I needed to avoid the existing dependency
files because those code paths need to know the output destination of
the manifest in order to parse the Make dependency files; trying to
adapt this system is more complicated than just preprocessing each
file to extract dependency information directly.
MozReview-Commit-ID: 5m0SEqmhJMM
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -11,16 +11,17 @@ import sys
from mozpack.errors import errors
from mozpack.files import (
BaseFile,
Dest,
)
import mozpack.path as mozpath
import errno
from collections import (
+ defaultdict,
Counter,
OrderedDict,
)
import concurrent.futures as futures
class FileRegistry(object):
'''
@@ -148,16 +149,44 @@ class FileRegistry(object):
'''
Return the set of directories required by the paths in the container,
in no particular order. The returned directories are relative to an
unspecified (virtual) root directory (and do not include said root
directory).
'''
return set(k for k, v in self._required_directories.items() if v > 0)
+ def output_to_inputs_tree(self):
+ '''
+ Return a dictionary mapping each output path to the set of its
+ required input paths.
+
+ All paths are normalized.
+ '''
+ tree = {}
+ for output, file in self:
+ output = mozpath.normpath(output)
+ tree[output] = set(mozpath.normpath(f) for f in file.inputs())
+ return tree
+
+ def input_to_outputs_tree(self):
+ '''
+ Return a dictionary mapping each input path to the set of
+ impacted output paths.
+
+ All paths are normalized.
+ '''
+ tree = defaultdict(set)
+ for output, file in self:
+ output = mozpath.normpath(output)
+ for input in file.inputs():
+ input = mozpath.normpath(input)
+ tree[input].add(output)
+ return dict(tree)
+
class FileRegistrySubtree(object):
'''A proxy class to give access to a subtree of an existing FileRegistry.
Note this doesn't implement the whole FileRegistry interface.'''
def __new__(cls, base, registry):
if not base:
return registry
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -229,16 +229,22 @@ class BaseFile(object):
@property
def mode(self):
'''
Return the file's unix mode, or None if it has no meaning.
'''
return None
+ def inputs(self):
+ '''
+ Return an iterable of the input file paths that impact this output file.
+ '''
+ raise NotImplementedError('BaseFile.inputs() not implemented.')
+
class File(BaseFile):
'''
File class for plain files.
'''
def __init__(self, path):
self.path = path
@@ -256,16 +262,19 @@ class File(BaseFile):
def read(self):
'''Return the contents of the file.'''
with open(self.path, 'rb') as fh:
return fh.read()
def size(self):
return os.stat(self.path).st_size
+ def inputs(self):
+ return (self.path,)
+
class ExecutableFile(File):
'''
File class for executable and library files on OS/2, OS/X and ELF systems.
(see mozpack.executables.is_executable documentation).
'''
def copy(self, dest, skip_if_older=True):
real_dest = dest
@@ -469,32 +478,46 @@ class ExistingFile(BaseFile):
if not self.required:
return
if not dest.exists():
errors.fatal("Required existing file doesn't exist: %s" %
dest.path)
+ def inputs(self):
+ return ()
+
class PreprocessedFile(BaseFile):
'''
File class for a file that is preprocessed. PreprocessedFile.copy() runs
the preprocessor on the file to create the output.
'''
def __init__(self, path, depfile_path, marker, defines, extra_depends=None,
silence_missing_directive_warnings=False):
self.path = path
self.depfile = depfile_path
self.marker = marker
self.defines = defines
self.extra_depends = list(extra_depends or [])
self.silence_missing_directive_warnings = \
silence_missing_directive_warnings
+ def inputs(self):
+ pp = Preprocessor(defines=self.defines, marker=self.marker)
+ pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
+
+ with open(self.path, 'rU') as input:
+ with open(os.devnull, 'w') as output:
+ pp.processFile(input=input, output=output)
+
+ # This always yields at least self.path.
+ return pp.includes
+
def copy(self, dest, skip_if_older=True):
'''
Invokes the preprocessor to create the destination file.
'''
if isinstance(dest, basestring):
dest = Dest(dest)
else:
assert isinstance(dest, Dest)
@@ -560,16 +583,19 @@ class GeneratedFile(BaseFile):
return BytesIO(self.content)
def read(self):
return self.content
def size(self):
return len(self.content)
+ def inputs(self):
+ return ()
+
class DeflatedFile(BaseFile):
'''
File class for members of a jar archive. DeflatedFile.copy() effectively
extracts the file from the jar archive.
'''
def __init__(self, file):
from mozpack.mozjar import JarFileReader