--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -14,17 +14,16 @@ from mozpack.copier import (
FileRegistry,
)
from mozpack.files import (
BaseFile,
FileFinder,
)
from mozpack.manifests import (
InstallManifest,
- InstallManifestNoSymlinks,
)
from mozbuild.util import DefinesAction
COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
'Added/updated {updated}; ' \
'Removed {rm_files} files and {rm_dirs} directories.'
@@ -51,23 +50,25 @@ def process_manifest(destdir, paths, tra
else:
# If tracking is enabled and there is no file, we don't want to
# be removing anything.
remove_unaccounted=False
remove_empty_directories=False
remove_all_directory_symlinks=False
- manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest
- manifest = manifest_cls()
+ manifest = InstallManifest()
for path in paths:
- manifest |= manifest_cls(path=path)
+ manifest |= InstallManifest(path=path)
copier = FileCopier()
- manifest.populate_registry(copier, defines_override=defines)
+ link_policy = "copy" if no_symlinks else "symlink"
+ manifest.populate_registry(
+ copier, defines_override=defines, link_policy=link_policy
+ )
result = copier.copy(destdir,
remove_unaccounted=remove_unaccounted,
remove_all_directory_symlinks=remove_all_directory_symlinks,
remove_empty_directories=remove_empty_directories)
if track:
manifest.write(path=track)
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -71,22 +71,22 @@ class FasterMakeBackend(CommonBackend, P
elif '*' in f:
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
prefix = ''.join(_prefix(f.full_path))
self._install_manifests[obj.install_target] \
- .add_pattern_symlink(
+ .add_pattern_link(
prefix,
f.full_path[len(prefix):],
mozpath.join(path, f.target_basename))
else:
- self._install_manifests[obj.install_target].add_symlink(
+ self._install_manifests[obj.install_target].add_link(
f.full_path,
mozpath.join(path, f.target_basename)
)
if isinstance(f, ObjDirPath):
dep_target = 'install-%s' % obj.install_target
self._dependencies[dep_target].append(
mozpath.relpath(f.full_path,
self.environment.topobjdir))
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1005,17 +1005,17 @@ class RecursiveMakeBackend(CommonBackend
def _handle_idl_manager(self, manager):
build_files = self._install_manifests['xpidl']
for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done'):
build_files.add_optional_exists(p)
for idl in manager.idls.values():
- self._install_manifests['dist_idl'].add_symlink(idl['source'],
+ self._install_manifests['dist_idl'].add_link(idl['source'],
idl['basename'])
self._install_manifests['dist_include'].add_optional_exists('%s.h'
% idl['root'])
for module in manager.modules:
build_files.add_optional_exists(mozpath.join('.deps',
'%s.pp' % module))
@@ -1150,24 +1150,24 @@ class RecursiveMakeBackend(CommonBackend
self.backend_input_files.add(mozpath.join(obj.topsrcdir,
source))
# Don't allow files to be defined multiple times unless it is allowed.
# We currently allow duplicates for non-test files or test files if
# the manifest is listed as a duplicate.
for source, (dest, is_test) in obj.installs.items():
try:
- self._install_manifests['_test_files'].add_symlink(source, dest)
+ self._install_manifests['_test_files'].add_link(source, dest)
except ValueError:
if not obj.dupe_manifest and is_test:
raise
for base, pattern, dest in obj.pattern_installs:
try:
- self._install_manifests['_test_files'].add_pattern_symlink(base,
+ self._install_manifests['_test_files'].add_pattern_link(base,
pattern, dest)
except ValueError:
if not obj.dupe_manifest:
raise
for dest in obj.external_installs:
try:
self._install_manifests['_test_files'].add_optional_exists(dest)
@@ -1385,21 +1385,21 @@ class RecursiveMakeBackend(CommonBackend
if not isinstance(f, ObjDirPath):
if '*' in f:
if f.startswith('/') or isinstance(f, AbsolutePath):
basepath, wild = os.path.split(f.full_path)
if '*' in basepath:
raise Exception("Wildcards are only supported in the filename part of "
"srcdir-relative or absolute paths.")
- install_manifest.add_pattern_symlink(basepath, wild, path)
+ install_manifest.add_pattern_link(basepath, wild, path)
else:
- install_manifest.add_pattern_symlink(f.srcdir, f, path)
+ install_manifest.add_pattern_link(f.srcdir, f, path)
else:
- install_manifest.add_symlink(f.full_path, dest)
+ install_manifest.add_link(f.full_path, dest)
else:
install_manifest.add_optional_exists(dest)
backend_file.write('%s_FILES += %s\n' % (
target_var, self._pretty_path(f, backend_file)))
have_objdir_files = True
if have_objdir_files:
tier = 'export' if obj.install_target == 'dist/include' else 'misc'
self._no_skip[tier].add(backend_file.relobjdir)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -583,19 +583,19 @@ class TestRecursiveMakeBackend(BackendTe
'child/another-file.sjs']))
for key in test_installs.keys():
self.assertIn(key, test_installs)
synthesized_manifest = InstallManifest()
for item, installs in test_installs.items():
for install_info in installs:
if len(install_info) == 3:
- synthesized_manifest.add_pattern_symlink(*install_info)
+ synthesized_manifest.add_pattern_link(*install_info)
if len(install_info) == 2:
- synthesized_manifest.add_symlink(*install_info)
+ synthesized_manifest.add_link(*install_info)
self.assertEqual(len(synthesized_manifest), 3)
for item, info in synthesized_manifest._dests.items():
self.assertIn(item, m)
self.assertEqual(info, m._dests[item])
def test_xpidl_generation(self):
"""Ensure xpidl files and directories are written out."""
@@ -655,17 +655,17 @@ class TestRecursiveMakeBackend(BackendTe
self.assertFalse(os.path.exists(manifest_path))
def test_install_manifests_written(self):
env, objs = self._emit('stub0')
backend = RecursiveMakeBackend(env)
m = InstallManifest()
backend._install_manifests['testing'] = m
- m.add_symlink(__file__, 'self')
+ m.add_link(__file__, 'self')
backend.consume(objs)
man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
self.assertTrue(os.path.isdir(man_dir))
expected = ['testing']
for e in expected:
full = mozpath.join(man_dir, e)
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -439,19 +439,19 @@ def _resolve_installs(paths, topobjdir,
raise Exception('A cross-directory support file path noted in a '
'test manifest does not appear in any other manifest.\n "%s" '
'must appear in another test manifest to specify an install '
'for "!/%s".' % (path, path))
installs = resolved_installs[path]
for install_info in installs:
try:
if len(install_info) == 3:
- manifest.add_pattern_symlink(*install_info)
+ manifest.add_pattern_link(*install_info)
if len(install_info) == 2:
- manifest.add_symlink(*install_info)
+ manifest.add_link(*install_info)
except ValueError:
# A duplicate value here is pretty likely when running
# multiple directories at once, and harmless.
pass
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
"""Installs the requested test files to the objdir. This is invoked by
test runners to avoid installing tens of thousands of test files when
@@ -490,19 +490,19 @@ def install_test_files(topsrcdir, topobj
manifest_dir,
out_dir)
manifest = InstallManifest()
for source, dest in set(install_info.installs):
if dest in install_info.external_installs:
continue
- manifest.add_symlink(source, dest)
+ manifest.add_link(source, dest)
for base, pattern, dest in install_info.pattern_installs:
- manifest.add_pattern_symlink(base, pattern, dest)
+ manifest.add_pattern_link(base, pattern, dest)
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
# Harness files are treated as a monolith and installed each time we run tests.
# Fortunately there are not very many.
manifest |= InstallManifest(mozpath.join(topobjdir,
'_build_manifests',
'install', tests_root))
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -8,16 +8,17 @@ from contextlib import contextmanager
import json
from .files import (
AbsoluteSymlinkFile,
ExistingFile,
File,
FileFinder,
GeneratedFile,
+ HardlinkFile,
PreprocessedFile,
)
import mozpack.path as mozpath
# This probably belongs in a more generic module. Where?
@contextmanager
def _auto_fileobj(path, fileobj, mode='r'):
@@ -50,32 +51,32 @@ class InstallManifest(object):
The manifest defines source paths, destination paths, and a mechanism by
which the destination file should come into existence.
Entries in the manifest correspond to the following types:
copy -- The file specified as the source path will be copied to the
destination path.
- symlink -- The destination path will be a symlink to the source path.
- If symlinks are not supported, a copy will be performed.
+ link -- The destination path will be a symlink or hardlink to the source
+ path. If symlinks are not supported, a copy will be performed.
exists -- The destination path is accounted for and won't be deleted by
the FileCopier. If the destination path doesn't exist, an error is
raised.
optional -- The destination path is accounted for and won't be deleted by
the FileCopier. No error is raised if the destination path does not
exist.
- patternsymlink -- Paths matched by the expression in the source path
- will be symlinked to the destination directory.
+ patternlink -- Paths matched by the expression in the source path
+ will be symlinked or hardlinked to the destination directory.
- patterncopy -- Similar to patternsymlink except files are copied, not
- symlinked.
+ patterncopy -- Similar to patternlink except files are copied, not
+ symlinked/hardlinked.
preprocess -- The file specified at the source path will be run through
the preprocessor, and the output will be written to the destination
path.
content -- The destination file will be created with the given content.
Version 1 of the manifest was the initial version.
@@ -86,21 +87,21 @@ class InstallManifest(object):
"""
CURRENT_VERSION = 5
FIELD_SEPARATOR = '\x1f'
# Negative values are reserved for non-actionable items, that is, metadata
# that doesn't describe files in the destination.
- SYMLINK = 1
+ LINK = 1
COPY = 2
REQUIRED_EXISTS = 3
OPTIONAL_EXISTS = 4
- PATTERN_SYMLINK = 5
+ PATTERN_LINK = 5
PATTERN_COPY = 6
PREPROCESS = 7
CONTENT = 8
def __init__(self, path=None, fileobj=None):
"""Create a new InstallManifest entry.
If path is defined, the manifest will be populated with data from the
@@ -127,19 +128,19 @@ class InstallManifest(object):
for line in fileobj:
line = line.rstrip()
fields = line.split(self.FIELD_SEPARATOR)
record_type = int(fields[0])
- if record_type == self.SYMLINK:
+ if record_type == self.LINK:
dest, source = fields[1:]
- self.add_symlink(source, dest)
+ self.add_link(source, dest)
continue
if record_type == self.COPY:
dest, source = fields[1:]
self.add_copy(source, dest)
continue
if record_type == self.REQUIRED_EXISTS:
@@ -147,19 +148,19 @@ class InstallManifest(object):
self.add_required_exists(path)
continue
if record_type == self.OPTIONAL_EXISTS:
_, path = fields
self.add_optional_exists(path)
continue
- if record_type == self.PATTERN_SYMLINK:
+ if record_type == self.PATTERN_LINK:
_, base, pattern, dest = fields[1:]
- self.add_pattern_symlink(base, pattern, dest)
+ self.add_pattern_link(base, pattern, dest)
continue
if record_type == self.PATTERN_COPY:
_, base, pattern, dest = fields[1:]
self.add_pattern_copy(base, pattern, dest)
continue
if record_type == self.PREPROCESS:
@@ -242,22 +243,22 @@ class InstallManifest(object):
for dest in sorted(self._dests):
entry = self._dests[dest]
parts = ['%d' % entry[0], dest]
parts.extend(entry[1:])
fh.write('%s\n' % self.FIELD_SEPARATOR.join(
p.encode('utf-8') for p in parts))
- def add_symlink(self, source, dest):
- """Add a symlink to this manifest.
+ def add_link(self, source, dest):
+ """Add a link to this manifest.
- dest will be a symlink to source.
+ dest will be either a symlink or hardlink to source.
"""
- self._add_entry(dest, (self.SYMLINK, source))
+ self._add_entry(dest, (self.LINK, source))
def add_copy(self, source, dest):
"""Add a copy to this manifest.
source will be copied to dest.
"""
self._add_entry(dest, (self.COPY, source))
@@ -272,35 +273,36 @@ class InstallManifest(object):
"""Record that a destination file may exist.
This effectively prevents the listed file from being deleted. Unlike a
"required exists" file, files of this type do not raise errors if the
destination file does not exist.
"""
self._add_entry(dest, (self.OPTIONAL_EXISTS,))
- def add_pattern_symlink(self, base, pattern, dest):
- """Add a pattern match that results in symlinks being created.
+ def add_pattern_link(self, base, pattern, dest):
+ """Add a pattern match that results in links being created.
A ``FileFinder`` will be created with its base set to ``base``
and ``FileFinder.find()`` will be called with ``pattern`` to discover
- source files. Each source file will be symlinked under ``dest``.
+ source files. Each source file will be either symlinked or hardlinked
+ under ``dest``.
Filenames under ``dest`` are constructed by taking the path fragment
after ``base`` and concatenating it with ``dest``. e.g.
<base>/foo/bar.h -> <dest>/foo/bar.h
"""
self._add_entry(mozpath.join(base, pattern, dest),
- (self.PATTERN_SYMLINK, base, pattern, dest))
+ (self.PATTERN_LINK, base, pattern, dest))
def add_pattern_copy(self, base, pattern, dest):
"""Add a pattern match that results in copies.
- See ``add_pattern_symlink()`` for usage.
+ See ``add_pattern_link()`` for usage.
"""
self._add_entry(mozpath.join(base, pattern, dest),
(self.PATTERN_COPY, base, pattern, dest))
def add_preprocess(self, source, dest, deps, marker='#', defines={},
silence_missing_directive_warnings=False):
"""Add a preprocessed file to this manifest.
@@ -324,53 +326,69 @@ class InstallManifest(object):
))
def _add_entry(self, dest, entry):
if dest in self._dests:
raise ValueError('Item already in manifest: %s' % dest)
self._dests[dest] = entry
- def populate_registry(self, registry, defines_override={}):
+ def populate_registry(self, registry, defines_override={},
+ link_policy='symlink'):
"""Populate a mozpack.copier.FileRegistry instance with data from us.
The caller supplied a FileRegistry instance (or at least something that
conforms to its interface) and that instance is populated with data
from this manifest.
Defines can be given to override the ones in the manifest for
preprocessing.
+
+ The caller can set a link policy. This determines whether symlinks,
+ hardlinks, or copies are used for LINK and PATTERN_LINK.
"""
+ assert link_policy in ("symlink", "hardlink", "copy")
for dest in sorted(self._dests):
entry = self._dests[dest]
install_type = entry[0]
- if install_type == self.SYMLINK:
- registry.add(dest, AbsoluteSymlinkFile(entry[1]))
+ if install_type == self.LINK:
+ if link_policy == "symlink":
+ cls = AbsoluteSymlinkFile
+ elif link_policy == "hardlink":
+ cls = HardlinkFile
+ else:
+ cls = File
+ registry.add(dest, cls(entry[1]))
continue
if install_type == self.COPY:
registry.add(dest, File(entry[1]))
continue
if install_type == self.REQUIRED_EXISTS:
registry.add(dest, ExistingFile(required=True))
continue
if install_type == self.OPTIONAL_EXISTS:
registry.add(dest, ExistingFile(required=False))
continue
- if install_type in (self.PATTERN_SYMLINK, self.PATTERN_COPY):
+ if install_type in (self.PATTERN_LINK, self.PATTERN_COPY):
_, base, pattern, dest = entry
finder = FileFinder(base)
paths = [f[0] for f in finder.find(pattern)]
- if install_type == self.PATTERN_SYMLINK:
- cls = AbsoluteSymlinkFile
+ if install_type == self.PATTERN_LINK:
+ if link_policy == "symlink":
+ cls = AbsoluteSymlinkFile
+ elif link_policy == "hardlink":
+ cls = HardlinkFile
+ else:
+ cls = File
else:
cls = File
for path in paths:
source = mozpath.join(base, path)
registry.add(mozpath.join(dest, path), cls(source))
continue
@@ -392,28 +410,8 @@ class InstallManifest(object):
# GeneratedFile expect the buffer interface, which the unicode
# type doesn't have, so encode to a str.
content = self._decode_field_entry(entry[1]).encode('utf-8')
registry.add(dest, GeneratedFile(content))
continue
raise Exception('Unknown install type defined in manifest: %d' %
install_type)
-
-
-class InstallManifestNoSymlinks(InstallManifest):
- """Like InstallManifest, but files are never installed as symbolic links.
- Instead, they are always copied.
- """
-
- def add_symlink(self, source, dest):
- """A wrapper that accept symlink entries and install file copies.
-
- source will be copied to dest.
- """
- self.add_copy(source, dest)
-
- def add_pattern_symlink(self, base, pattern, dest):
- """A wrapper that accepts symlink patterns and installs file copies.
-
- Files discovered with ``pattern`` will be copied to ``dest``.
- """
- self.add_pattern_copy(base, pattern, dest)
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -27,65 +27,65 @@ class TestInstallManifest(TestWithTmpDir
def test_malformed(self):
f = self.tmppath('manifest')
open(f, 'wb').write('junk\n')
with self.assertRaises(UnreadableInstallManifest):
m = InstallManifest(f)
def test_adds(self):
m = InstallManifest()
- m.add_symlink('s_source', 's_dest')
+ m.add_link('s_source', 's_dest')
m.add_copy('c_source', 'c_dest')
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
- m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
+ m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
m.add_preprocess('p_source', 'p_dest', 'p_source.pp')
m.add_content('content', 'content')
self.assertEqual(len(m), 8)
self.assertIn('s_dest', m)
self.assertIn('c_dest', m)
self.assertIn('p_dest', m)
self.assertIn('e_dest', m)
self.assertIn('o_dest', m)
self.assertIn('content', m)
with self.assertRaises(ValueError):
- m.add_symlink('s_other', 's_dest')
+ m.add_link('s_other', 's_dest')
with self.assertRaises(ValueError):
m.add_copy('c_other', 'c_dest')
with self.assertRaises(ValueError):
m.add_preprocess('p_other', 'p_dest', 'p_other.pp')
with self.assertRaises(ValueError):
m.add_required_exists('e_dest')
with self.assertRaises(ValueError):
m.add_optional_exists('o_dest')
with self.assertRaises(ValueError):
- m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
+ m.add_pattern_link('ps_base', 'ps/*', 'ps_dest')
with self.assertRaises(ValueError):
m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
with self.assertRaises(ValueError):
m.add_content('content', 'content')
def _get_test_manifest(self):
m = InstallManifest()
- m.add_symlink(self.tmppath('s_source'), 's_dest')
+ m.add_link(self.tmppath('s_source'), 's_dest')
m.add_copy(self.tmppath('c_source'), 'c_dest')
m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'})
m.add_required_exists('e_dest')
m.add_optional_exists('o_dest')
- m.add_pattern_symlink('ps_base', '*', 'ps_dest')
+ m.add_pattern_link('ps_base', '*', 'ps_dest')
m.add_pattern_copy('pc_base', '**', 'pc_dest')
m.add_content('the content\non\nmultiple lines', 'content')
return m
def test_serialization(self):
m = self._get_test_manifest()
@@ -130,27 +130,27 @@ class TestInstallManifest(TestWithTmpDir
with open('%s/base/foo/file1' % source, 'a'):
pass
with open('%s/base/foo/file2' % source, 'a'):
pass
m = InstallManifest()
- m.add_pattern_symlink('%s/base' % source, '**', 'dest')
+ m.add_pattern_link('%s/base' % source, '**', 'dest')
c = FileCopier()
m.populate_registry(c)
self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
def test_or(self):
m1 = self._get_test_manifest()
orig_length = len(m1)
m2 = InstallManifest()
- m2.add_symlink('s_source2', 's_dest2')
+ m2.add_link('s_source2', 's_dest2')
m2.add_copy('c_source2', 'c_dest2')
m1 |= m2
self.assertEqual(len(m2), 2)
self.assertEqual(len(m1), orig_length + 2)
self.assertIn('s_dest2', m1)
--- a/tools/docs/moztreedocs/__init__.py
+++ b/tools/docs/moztreedocs/__init__.py
@@ -91,26 +91,26 @@ class SphinxManager(object):
args.append(full)
args.extend(excludes)
sphinx.apidoc.main(args)
def _synchronize_docs(self):
m = InstallManifest()
- m.add_symlink(self._conf_py_path, 'conf.py')
+ m.add_link(self._conf_py_path, 'conf.py')
for dest, source in sorted(self._trees.items()):
source_dir = os.path.join(self._topsrcdir, source)
for root, dirs, files in os.walk(source_dir):
for f in files:
source_path = os.path.join(root, f)
rel_source = source_path[len(source_dir) + 1:]
- m.add_symlink(source_path, os.path.join(dest, rel_source))
+ m.add_link(source_path, os.path.join(dest, rel_source))
copier = FileCopier()
m.populate_registry(copier)
copier.copy(self._docs_dir)
with open(self._index_path, 'rb') as fh:
data = fh.read()
