vcssync: utility function to copy between working directories (bug 1357597); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Wed, 13 Sep 2017 13:27:25 -0700
changeset 11666 3c202d92e09a571f0facb0abbedf3e9fdd3d8f42
parent 11665 b0d4ede2218c3f5e9a8c4e8f7d1484f9fc46e536
child 11667 48377dec86d6c31a40381b4ddf4db1a72df493ab
push id1785
push usergszorc@mozilla.com
push dateFri, 15 Sep 2017 01:22:24 +0000
reviewersglob
bugs1357597
vcssync: utility function to copy between working directories (bug 1357597); r?glob Synchronizing the contents of working directories is not trivial because of ignored and untracked files. Common tools like rsync aren't aware of VCS state and will likely copy files it shouldn't. So, we implement our own recursive directory synchronization primitive that is VCS aware. We only support Git to Mercurial at the moment. But the other direction is trivial to implement. MozReview-Commit-ID: Dgg24sFNDO6
vcssync/mozvcssync/cli.py
vcssync/mozvcssync/git2hg.py
vcssync/mozvcssync/util.py
vcssync/prod-requirements.txt
vcssync/setup.py
vcssync/tests/test-copy-workdir-git-to-hg.t
vcssync/tests/test-synchronize-directories.t
--- a/vcssync/mozvcssync/cli.py
+++ b/vcssync/mozvcssync/cli.py
@@ -7,18 +7,20 @@ from __future__ import absolute_import, 
 import argparse
 import logging
 import os
 import subprocess
 import sys
 
 import dulwich.repo
 import hglib
+import pathlib2 as pathlib
 
 from .git2hg import (
+    copy_git_workdir_to_hg_workdir,
     linearize_git_repo_to_hg,
 )
 from .gitrewrite import (
     RewriteError,
     commit_metadata_rewriter,
 )
 from .gitrewrite.linearize import (
     linearize_git_repo,
@@ -26,16 +28,19 @@ from .gitrewrite.linearize import (
 from .overlay import (
     overlay_hg_repos,
     PushRaceError,
     PushRemoteFail,
 )
 from .servo_backout import (
     backout_servo_pr,
 )
+from .util import (
+    synchronize_directories,
+)
 
 logger = logging.getLogger(__name__)
 
 
 LINEARIZE_GIT_ARGS = [
     (('--exclude-dir',), dict(action='append', dest='exclude_dirs',
                               help='Directory to exclude from rewritten '
                                    'history')),
@@ -317,8 +322,61 @@ def servo_backout_pr_cli():
             args.github_repo_path,
             args.author,
             args.tracking_s3_upload_url,
             args.revision,
         )
     except Exception as e:
         logger.exception(e)
         sys.exit(1)
+
+
+def test_synchronize_directories():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--map', action='append',
+                        help='source:dest directory mapping')
+    parser.add_argument('source')
+    parser.add_argument('dest')
+
+    args = parser.parse_args()
+
+    source_files = []
+    base = pathlib.PosixPath(args.source)
+    for p in base.glob('**/*'):
+        if p == base:
+            continue
+
+        if p.is_dir():
+            continue
+
+        source_files.append(bytes(p.relative_to(base)))
+
+    dir_map = {}
+    for e in args.map or []:
+        k, v = e.split(':')
+        dir_map[k] = v
+
+    try:
+        synchronize_directories(args.source, source_files, args.dest, dir_map)
+    except Exception as e:
+        print('%s: %s' % (e.__class__.__name__, e))
+        sys.exit(1)
+
+
+def test_copy_workdir_git_to_hg():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--map', action='append',
+                        help='source:dest directory mapping')
+    parser.add_argument('source')
+    parser.add_argument('dest')
+
+    args = parser.parse_args()
+
+    dir_map = {}
+    for e in args.map or []:
+        k, v = e.split(':')
+        dir_map[k] = v
+
+    try:
+        copy_git_workdir_to_hg_workdir(args.source, args.dest, dir_map)
+    except Exception as e:
+        print('%s: %s' % (e.__class__.__name__, e))
+        sys.exit(1)
--- a/vcssync/mozvcssync/git2hg.py
+++ b/vcssync/mozvcssync/git2hg.py
@@ -16,16 +16,17 @@ import hglib
 from .gitrewrite import (
     commit_metadata_rewriter,
 )
 from .gitrewrite.linearize import (
     linearize_git_repo,
 )
 from .util import (
     monitor_hg_repo,
+    synchronize_directories,
 )
 
 
 logger = logging.getLogger(__name__)
 
 
 def source_commits_in_map_file(path, commits):
     """Determine whether all source commits are present in a map file.
@@ -286,8 +287,45 @@ def linearize_git_repo_to_hg(git_source_
 
     # TODO so hacky. Relies on credentials in the environment.
     if shamap_s3_upload_url and shamap_changed:
         subprocess.check_call([
             b'aws', b's3', b'cp', rev_map, shamap_s3_upload_url
         ])
 
     return result
+
+
+def copy_git_workdir_to_hg_workdir(git_repo_path, hg_repo_path, dir_map=None):
+    """Copy files between Git and Mercurial working directories.
+
+    Given paths to Git and Mercurial repos that have working directories,
+    synchronize the contents of the Git working directory to the Mercurial
+    working directory, optionally remapping paths to directories using
+    ``dir_map``.
+
+    See ``util.synchronize_directories()`` for behavior of ``dir_map``.
+
+    Only files tracked by Git are copied. This therefore differs from a pure
+    filesystem recursive copy.
+
+    The changes to the Mercurial working directory are left uncommitted.
+
+    Returns a bool indicating if there are changes to be committed.
+    """
+    git_files = subprocess.check_output([b'git', b'ls-files', b'-z'],
+                                        cwd=git_repo_path)
+    git_files = [f for f in git_files.split(b'\0') if f]
+
+    synchronize_directories(git_repo_path, git_files, hg_repo_path,
+                            dir_map=dir_map)
+
+    have_changes = False
+
+    configs = [b'extensions.automv=']
+    with hglib.open(hg_repo_path, encoding='utf-8', configs=configs) as hrepo:
+        hrepo.addremove()
+        for code, path in hrepo.status():
+            if code in (b'M', b'A', b'R'):
+                have_changes = True
+                break
+
+    return have_changes
--- a/vcssync/mozvcssync/util.py
+++ b/vcssync/mozvcssync/util.py
@@ -10,16 +10,18 @@ import hashlib
 import logging
 import os
 import pipes
 import shutil
 import sys
 
 import github3
 import hglib
+import pathlib2 as pathlib
+import scandir
 
 
 def run_hg(logger, client, args):
     """Run a Mercurial command through hgclient and log output."""
     logger.warn('executing: hg %s' % ' '.join(map(pipes.quote, args)))
     out = hglib.util.BytesIO()
 
     def write(data):
@@ -170,8 +172,149 @@ def apply_changes_from_list(logger, sour
 
 def test_apply_changes_from_list():
     logger = logging.getLogger('mozvcssync')
     logger.addHandler(logging.StreamHandler(sys.stdout))
     logger.setLevel(logging.INFO)
     source_path, dest_path, filelist = sys.argv[1::]
     apply_changes_from_list(logger,
                             source_path, dest_path, filelist.split(','))
+
+
+def resolve_all_dirs(paths):
+    """Resolve all relative directories given an iterable of paths.
+
+    Returns a set of ``pathlib.PurePosixPath`` instances consisting of all
+    directories (including parents) seen in the passed set of paths.
+
+    It is assumed that all paths are files - not directories.
+
+    The root directory (``/`` if absolute paths are passed and ``.`` if relative
+    paths) will be present in the returned set.
+    """
+    dirs = set()
+    for f in paths:
+        p = pathlib.PurePosixPath(f)
+        for parent in p.parents:
+            dirs.add(parent)
+
+    return dirs
+
+
+def rmtree_except_vcs(path):
+    """Recursively delete a directory except for VCS directories.
+
+    The specified base directory is not itself deleted.
+    """
+    try:
+        it = scandir.scandir(path)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return
+        raise
+
+    for entry in it:
+        if entry.is_dir():
+            if entry.name in ('.hg', '.git'):
+                continue
+
+            shutil.rmtree(entry.path)
+        else:
+            pathlib.Path(entry.path).unlink()
+
+
+def synchronize_directories(source_base, source_files, dest_base, dir_map=None):
+    """Synchronize the contents of directories.
+
+    This function is meant to be used to synchronize the contents of Git and
+    Mercurial working directories.
+
+    It takes two pairs of (dir, files) arguments, defining the base directory
+    and an iterable of files in those directories.
+
+    ``dir_map`` defines a mapping of directory paths in the source to paths
+    in the dest. If provided, only the destination directories will be possibly
+    mutated during execution. Otherwise, the entire set of files is
+    synchronized.
+
+    Files in the destination that aren't accounted for in the source are
+    removed. New files and file modifications are performed as necessary. File
+    permissions and owners are preserved.
+
+    Symlinks are not yet handled properly in all cases. If a symlink exists in
+    the source, behavior is not defined.
+    """
+    source_files = set(source_files)
+
+    # Do a full sync. We could probably use rsync. But since our source files
+    # are defined explicitly and may not resemble actual filesystem contents,
+    # we'd have to use an rsync rules file. At the point we have that
+    # complexity, it is only slightly more difficult to inline in Python.
+    if not dir_map:
+        rmtree_except_vcs(dest_base)
+
+        rel_dirs = resolve_all_dirs(source_files)
+        rel_dirs.remove(pathlib.PurePosixPath('.'))
+
+        for rel_dir in sorted(rel_dirs, key=lambda x: len(bytes(x))):
+            p = pathlib.Path(dest_base, rel_dir)
+            p.mkdir()
+
+        for f in sorted(source_files):
+            source = pathlib.Path(source_base, f)
+            dest = pathlib.Path(dest_base, f)
+            shutil.copy2(bytes(source), bytes(dest))
+
+        return
+
+    # We must have a directory map.
+    #
+    # First some validation.
+
+    if len(set(dir_map.values())) != len(dir_map):
+        raise ValueError('cannot map multiple inputs to same output')
+
+    # Let's find the relevant source files and map them to outputs.
+
+    file_map = {}
+    dest_dirs = set()
+
+    for f in source_files:
+        p = pathlib.PurePosixPath(f)
+
+        # Traverse the parent directories looking for an entry in the directory
+        # map. If we find an entry, apply the mapping. Otherwise we drop this
+        # file.
+        for d in p.parents:
+            if bytes(d) not in dir_map:
+                continue
+
+            # We have a map entry. Rewrite this path component to the new one
+            rel_path = p.relative_to(d)
+            dest_path = pathlib.PurePosixPath(dir_map[bytes(d)], rel_path)
+
+            file_map[p] = dest_path
+            dest_dirs.add(dest_path.parent)
+
+    # Nuke our destination directories to ensure they are pristine. This
+    # behavior could be configurable to do an overlay instead. We could also
+    # only sync if needed. But it is easier to just nuke and repave.
+
+    # If synchronizing to the root directory, we can't delete the root because
+    # it may have VCS directories. So handle it specially.
+    for p in sorted(dest_dirs):
+        rmtree_except_vcs(bytes(pathlib.Path(dest_base, p)))
+
+    for p in sorted(resolve_all_dirs(file_map.values()),
+                    key=lambda x: len(bytes(x))):
+        p = pathlib.Path(dest_base, p)
+        try:
+            p.mkdir()
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+    # And finally copy the file content over.
+    for source, dest in sorted(file_map.items()):
+        source = pathlib.Path(source_base, source)
+        dest = pathlib.Path(dest_base, dest)
+
+        shutil.copy2(bytes(source), bytes(dest))
--- a/vcssync/prod-requirements.txt
+++ b/vcssync/prod-requirements.txt
@@ -13,19 +13,25 @@ github3.py==0.9.6 \
     --hash=sha256:650d31dbc3f3290ea56b18cfd0e72e00bbbd6777436578865a7e45b496f09e4c
 
 kombu==3.0.37 \
     --hash=sha256:7ceab743e3e974f3e5736082e8cc514c009e254e646d6167342e0e192aee81a6
 
 Mercurial==4.2.3 \
     --hash=sha256:04908fc7d89e5810edf3d2762f5aecce5b5c0cb8534f3dbff7d0d848d11ff7ac
 
+pathlib2==2.3.0 \
+    --hash=sha256:db3e43032d23787d3e9aec8c7ef1e0d2c3c589d5f303477661ebda2ca6d4bfba
+
 python-hglib==1.7 \
     --hash=sha256:0dc087d15b774cda82d3c8096fb0e514caeb2ddb60eed38e9056b16e279ba3c5
 
 requests==2.13.0 \
     --hash=sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb
 
+scandir==1.5 \
+    --hash=sha256:c2612d1a487d80fb4701b4a91ca1b8f8a695b1ae820570815e85e8c8b23f1283
+
 uritemplate==3.0.0 \
     --hash=sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd
 
 uritemplate.py==3.0.2 \
     --hash=sha256:a0c459569e80678c473175666e0d1b3af5bc9a13f84463ec74f808f3dd12ca47
--- a/vcssync/setup.py
+++ b/vcssync/setup.py
@@ -10,16 +10,18 @@ console_scripts = [
     'servo-pulse-listen=mozvcssync.servo:pulse_daemon',
     'test-apply-changes=mozvcssync.util:test_apply_changes_from_list',
 ]
 
 # These commands are really only useful for testing. So don't expose them by
 # default.
 if 'VCSSYNC_ENABLE_TESTING_COMMANDS' in os.environ:
     console_scripts.extend([
+        'test-synchronize-directories=mozvcssync.cli:test_synchronize_directories',
+        'test-copy-workdir-git-to-hg=mozvcssync.cli:test_copy_workdir_git_to_hg',
     ])
 
 # ansible/roles/vcs-sync/defaults/main.yml must be updated if this package's
 # version number is changed.
 
 setup(
     name='mozvcssync',
     version='0.1',
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-copy-workdir-git-to-hg.t
@@ -0,0 +1,170 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+
+Invalid source repository
+
+  $ mkdir bad-source
+  $ test-copy-workdir-git-to-hg bad-source irrelevant
+  fatal: Not a git repository (or any of the parent directories): .git
+  CalledProcessError: Command '['git', 'ls-files', '-z']' returned non-zero exit status 128
+  [1]
+
+Populate Git repository
+
+  $ git init source0
+  Initialized empty Git repository in $TESTTMP/source0/.git/
+  $ cd source0
+  $ echo file0 > file0
+  $ git add file0
+  $ git commit -m initial
+  [master (root-commit) aec8d6b] initial
+   1 file changed, 1 insertion(+)
+   create mode 100644 file0
+
+  $ mkdir -p dir0/subdir0 dir0/subdir1 dir2 dir3/subdir2/subsubdir0
+  $ echo file1 > dir0/file1
+  $ echo file2 > dir0/subdir0/file2
+  $ echo file3 > dir0/subdir0/file3
+  $ echo file4 > dir0/subdir1/file4
+  $ echo file5 > dir2/file5
+  $ echo file6 > dir3/subdir2/subsubdir0/file6
+
+  $ git add -A
+  $ git commit -m 'add more files'
+  [master c02e407] add more files
+   6 files changed, 6 insertions(+)
+   create mode 100644 dir0/file1
+   create mode 100644 dir0/subdir0/file2
+   create mode 100644 dir0/subdir0/file3
+   create mode 100644 dir0/subdir1/file4
+   create mode 100644 dir2/file5
+   create mode 100644 dir3/subdir2/subsubdir0/file6
+
+  $ touch ignored dir0/ignored dir2/ignored
+  $ cat > .gitignore << EOF
+  > ignored
+  > EOF
+  $ git add .gitignore
+  $ git commit -m 'add ignore rules'
+  [master 837b388] add ignore rules
+   1 file changed, 1 insertion(+)
+   create mode 100644 .gitignore
+
+  $ cd ..
+
+Invalid destination repository
+
+  $ mkdir bad-dest
+  $ test-copy-workdir-git-to-hg source0 bad-dest
+  ServerError: 
+  [1]
+
+Full synchronization works
+
+  $ hg init empty-dest
+  $ test-copy-workdir-git-to-hg source0 empty-dest
+
+  $ hg -R empty-dest status
+  A .gitignore
+  A dir0/file1
+  A dir0/subdir0/file2
+  A dir0/subdir0/file3
+  A dir0/subdir1/file4
+  A dir2/file5
+  A dir3/subdir2/subsubdir0/file6
+  A file0
+
+Synchronizing a specific directory works
+
+  $ hg init rename-directory
+  $ test-copy-workdir-git-to-hg source0 rename-directory --map dir0:renamed
+
+  $ hg -R rename-directory status
+  A renamed/file1
+  A renamed/subdir0/file2
+  A renamed/subdir0/file3
+  A renamed/subdir1/file4
+
+Synchronizing a sub-directory to the root directory works
+
+  $ hg init subdir-to-root
+  $ test-copy-workdir-git-to-hg source0 subdir-to-root --map dir0:.
+
+  $ hg -R subdir-to-root status
+  A file1
+  A subdir0/file2
+  A subdir0/file3
+  A subdir1/file4
+
+Moving entire repository to a sub-directory works
+
+  $ hg init root-to-subdir
+  $ test-copy-workdir-git-to-hg source0 root-to-subdir --map .:newparent
+
+  $ hg -R root-to-subdir status
+  A newparent/.gitignore
+  A newparent/dir0/file1
+  A newparent/dir0/subdir0/file2
+  A newparent/dir0/subdir0/file3
+  A newparent/dir0/subdir1/file4
+  A newparent/dir2/file5
+  A newparent/dir3/subdir2/subsubdir0/file6
+  A newparent/file0
+
+Extra files in dest are scheduled for deletion
+
+  $ hg init extra-files
+  $ cd extra-files
+  $ mkdir -p dir0/subdir0
+  $ touch dir0/extra-file dir0/subdir0/extra-file
+  $ hg -q commit -A -m initial
+  $ cd ..
+
+  $ test-copy-workdir-git-to-hg source0 extra-files --map dir0:dir0
+  $ hg -R extra-files status
+  A dir0/file1
+  A dir0/subdir0/file2
+  A dir0/subdir0/file3
+  A dir0/subdir1/file4
+  R dir0/extra-file
+  R dir0/subdir0/extra-file
+
+Rename detection works
+
+  $ git clone source0 source-rename
+  Cloning into 'source-rename'...
+  done.
+  $ hg init rename-detection
+  $ test-copy-workdir-git-to-hg source-rename rename-detection
+  $ hg -R rename-detection commit -m initial
+
+  $ cd source-rename
+  $ git mv dir0 dir0-renamed
+  $ git mv dir2/file5 dir2/file5-renamed
+  $ git commit -m 'perform renames'
+  [master 5ba3017] perform renames
+   5 files changed, 0 insertions(+), 0 deletions(-)
+   rename {dir0 => dir0-renamed}/file1 (100%)
+   rename {dir0 => dir0-renamed}/subdir0/file2 (100%)
+   rename {dir0 => dir0-renamed}/subdir0/file3 (100%)
+   rename {dir0 => dir0-renamed}/subdir1/file4 (100%)
+   rename dir2/{file5 => file5-renamed} (100%)
+  $ cd ..
+
+  $ test-copy-workdir-git-to-hg source-rename rename-detection
+
+  $ hg -R rename-detection status --copies
+  A dir0-renamed/file1
+    dir0/file1
+  A dir0-renamed/subdir0/file2
+    dir0/subdir0/file2
+  A dir0-renamed/subdir0/file3
+    dir0/subdir0/file3
+  A dir0-renamed/subdir1/file4
+    dir0/subdir1/file4
+  A dir2/file5-renamed
+    dir2/file5
+  R dir0/file1
+  R dir0/subdir0/file2
+  R dir0/subdir0/file3
+  R dir0/subdir1/file4
+  R dir2/file5
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-synchronize-directories.t
@@ -0,0 +1,323 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+
+Create a test source directory tree
+
+  $ mkdir source0
+  $ cd source0
+  $ mkdir -p dir0/subdir0 dir0/subdir1 dir0/subdir2 dir1/subdir3 dir1/subdir4 dir2 dir3/subdir5/subsubdir0
+  $ echo file0 > file0
+  $ echo file1 > dir0/file1
+  $ echo file2 > dir0/file2
+  $ chmod +x dir0/file2
+  $ echo file3 > dir0/subdir0/file3
+  $ echo file4 > dir0/subdir1/file4
+  $ echo file5 > dir0/subdir2/file5
+  $ echo file6 > dir0/subdir2/file6
+  $ echo file7 > dir1/subdir4/file7
+  $ echo file8 > dir2/file8
+  $ echo file9 > dir3/subdir5/subsubdir0/file9
+
+  $ cd ..
+
+A full synchronization to empty dest works
+
+  $ mkdir dest-full
+  $ test-synchronize-directories source0 dest-full
+
+  $ find dest-full | sort
+  dest-full
+  dest-full/dir0
+  dest-full/dir0/file1
+  dest-full/dir0/file2
+  dest-full/dir0/subdir0
+  dest-full/dir0/subdir0/file3
+  dest-full/dir0/subdir1
+  dest-full/dir0/subdir1/file4
+  dest-full/dir0/subdir2
+  dest-full/dir0/subdir2/file5
+  dest-full/dir0/subdir2/file6
+  dest-full/dir1
+  dest-full/dir1/subdir4
+  dest-full/dir1/subdir4/file7
+  dest-full/dir2
+  dest-full/dir2/file8
+  dest-full/dir3
+  dest-full/dir3/subdir5
+  dest-full/dir3/subdir5/subsubdir0
+  dest-full/dir3/subdir5/subsubdir0/file9
+  dest-full/file0
+
+Synchronization to hg repo doesn't delete .hg/
+
+  $ hg init dest-hg
+  $ test-synchronize-directories source0 dest-hg
+  $ find dest-hg | sort
+  dest-hg
+  dest-hg/.hg
+  dest-hg/.hg/00changelog.i
+  dest-hg/.hg/requires
+  dest-hg/.hg/store
+  dest-hg/dir0
+  dest-hg/dir0/file1
+  dest-hg/dir0/file2
+  dest-hg/dir0/subdir0
+  dest-hg/dir0/subdir0/file3
+  dest-hg/dir0/subdir1
+  dest-hg/dir0/subdir1/file4
+  dest-hg/dir0/subdir2
+  dest-hg/dir0/subdir2/file5
+  dest-hg/dir0/subdir2/file6
+  dest-hg/dir1
+  dest-hg/dir1/subdir4
+  dest-hg/dir1/subdir4/file7
+  dest-hg/dir2
+  dest-hg/dir2/file8
+  dest-hg/dir3
+  dest-hg/dir3/subdir5
+  dest-hg/dir3/subdir5/subsubdir0
+  dest-hg/dir3/subdir5/subsubdir0/file9
+  dest-hg/file0
+
+Synchronization to git repo doesn't delete .git/
+
+  $ git init dest-git
+  Initialized empty Git repository in $TESTTMP/dest-git/.git/
+  $ test-synchronize-directories source0 dest-git
+  $ find dest-git -maxdepth 2 | sort
+  dest-git
+  dest-git/.git
+  dest-git/.git/HEAD
+  dest-git/.git/branches
+  dest-git/.git/config
+  dest-git/.git/description
+  dest-git/.git/hooks
+  dest-git/.git/info
+  dest-git/.git/objects
+  dest-git/.git/refs
+  dest-git/dir0
+  dest-git/dir0/file1
+  dest-git/dir0/file2
+  dest-git/dir0/subdir0
+  dest-git/dir0/subdir1
+  dest-git/dir0/subdir2
+  dest-git/dir1
+  dest-git/dir1/subdir4
+  dest-git/dir2
+  dest-git/dir2/file8
+  dest-git/dir3
+  dest-git/dir3/subdir5
+  dest-git/file0
+
+Executable bit is preserved
+
+  $ test -x dest-full/dir0/file2
+
+Extra files in dest should be removed
+
+  $ mkdir -p dest-populated/dir0 dest-populated/dir0/subdir-extra dest-populated/dir-extra
+  $ echo extra > dest-populated/dir0/file-extra
+  $ echo extra > dest-populated/dir0/subdir-extra/file-extra
+  $ echo extra > dest-populated/dir-extra/file-extra
+
+  $ test-synchronize-directories source0 dest-populated
+
+  $ find dest-populated | sort
+  dest-populated
+  dest-populated/dir0
+  dest-populated/dir0/file1
+  dest-populated/dir0/file2
+  dest-populated/dir0/subdir0
+  dest-populated/dir0/subdir0/file3
+  dest-populated/dir0/subdir1
+  dest-populated/dir0/subdir1/file4
+  dest-populated/dir0/subdir2
+  dest-populated/dir0/subdir2/file5
+  dest-populated/dir0/subdir2/file6
+  dest-populated/dir1
+  dest-populated/dir1/subdir4
+  dest-populated/dir1/subdir4/file7
+  dest-populated/dir2
+  dest-populated/dir2/file8
+  dest-populated/dir3
+  dest-populated/dir3/subdir5
+  dest-populated/dir3/subdir5/subsubdir0
+  dest-populated/dir3/subdir5/subsubdir0/file9
+  dest-populated/file0
+
+Existing file content should be replaced by source content
+
+  $ mkdir -p dest-content/dir0 dest-content/dir0/subdir0
+  $ echo file1-modified > dest-content/dir0/file1
+  $ echo file3-modified > dest-content/dir0/subdir0/file3
+
+  $ test-synchronize-directories source0 dest-content
+
+  $ find dest-populated -type f | sort
+  dest-populated/dir0/file1
+  dest-populated/dir0/file2
+  dest-populated/dir0/subdir0/file3
+  dest-populated/dir0/subdir1/file4
+  dest-populated/dir0/subdir2/file5
+  dest-populated/dir0/subdir2/file6
+  dest-populated/dir1/subdir4/file7
+  dest-populated/dir2/file8
+  dest-populated/dir3/subdir5/subsubdir0/file9
+  dest-populated/file0
+
+  $ cat dest-content/dir0/file1
+  file1
+  $ cat dest-content/dir0/subdir0/file3
+  file3
+
+Executable bit should be cleared or set as appropriate
+
+  $ mkdir -p dest-executable/dir0
+  $ echo file1 > dest-executable/dir0/file1
+  $ chmod +x dest-executable/dir0/file1
+  $ echo file2 > dest-executable/dir0/file2
+
+  $ test-synchronize-directories source0 dest-executable
+  $ find dest-executable -type f | sort
+  dest-executable/dir0/file1
+  dest-executable/dir0/file2
+  dest-executable/dir0/subdir0/file3
+  dest-executable/dir0/subdir1/file4
+  dest-executable/dir0/subdir2/file5
+  dest-executable/dir0/subdir2/file6
+  dest-executable/dir1/subdir4/file7
+  dest-executable/dir2/file8
+  dest-executable/dir3/subdir5/subsubdir0/file9
+  dest-executable/file0
+
+  $ test -x dest-executable/dir0/file1
+  [1]
+  $ test -x dest-executable/dir0/file2
+
+Attempt to map multiple sources to same dest files
+
+  $ test-synchronize-directories source0 dest-bad --map dir0:same-dir --map dir1:same-dir
+  ValueError: cannot map multiple inputs to same output
+  [1]
+
+Only synchronize a sub-directory to the same sub-directory
+
+  $ mkdir dest-dir0
+  $ test-synchronize-directories source0 dest-dir0 --map dir0:dir0
+
+  $ find dest-dir0 | sort
+  dest-dir0
+  dest-dir0/dir0
+  dest-dir0/dir0/file1
+  dest-dir0/dir0/file2
+  dest-dir0/dir0/subdir0
+  dest-dir0/dir0/subdir0/file3
+  dest-dir0/dir0/subdir1
+  dest-dir0/dir0/subdir1/file4
+  dest-dir0/dir0/subdir2
+  dest-dir0/dir0/subdir2/file5
+  dest-dir0/dir0/subdir2/file6
+
+Synchronize multiple sibling directories
+
+  $ mkdir dest-siblingdirs
+  $ test-synchronize-directories source0 dest-siblingdirs --map dir0/subdir0:dir0/subdir0 --map dir0/subdir1:dir0/subdir1
+
+  $ find dest-siblingdirs | sort
+  dest-siblingdirs
+  dest-siblingdirs/dir0
+  dest-siblingdirs/dir0/subdir0
+  dest-siblingdirs/dir0/subdir0/file3
+  dest-siblingdirs/dir0/subdir1
+  dest-siblingdirs/dir0/subdir1/file4
+
+Move a directory to the root
+
+  $ mkdir dest-move-to-root
+  $ test-synchronize-directories source0 dest-move-to-root --map dir0:.
+
+  $ find dest-move-to-root | sort
+  dest-move-to-root
+  dest-move-to-root/file1
+  dest-move-to-root/file2
+  dest-move-to-root/subdir0
+  dest-move-to-root/subdir0/file3
+  dest-move-to-root/subdir1
+  dest-move-to-root/subdir1/file4
+  dest-move-to-root/subdir2
+  dest-move-to-root/subdir2/file5
+  dest-move-to-root/subdir2/file6
+
+Executable bit should be preserved
+
+  $ test -x dest-move-to-root/file2
+
+Move the root to a sub-directory
+
+  $ mkdir dest-move-to-child
+  $ test-synchronize-directories source0 dest-move-to-child --map .:newparent
+
+  $ find dest-move-to-child | sort
+  dest-move-to-child
+  dest-move-to-child/newparent
+  dest-move-to-child/newparent/dir0
+  dest-move-to-child/newparent/dir0/file1
+  dest-move-to-child/newparent/dir0/file2
+  dest-move-to-child/newparent/dir0/subdir0
+  dest-move-to-child/newparent/dir0/subdir0/file3
+  dest-move-to-child/newparent/dir0/subdir1
+  dest-move-to-child/newparent/dir0/subdir1/file4
+  dest-move-to-child/newparent/dir0/subdir2
+  dest-move-to-child/newparent/dir0/subdir2/file5
+  dest-move-to-child/newparent/dir0/subdir2/file6
+  dest-move-to-child/newparent/dir1
+  dest-move-to-child/newparent/dir1/subdir4
+  dest-move-to-child/newparent/dir1/subdir4/file7
+  dest-move-to-child/newparent/dir2
+  dest-move-to-child/newparent/dir2/file8
+  dest-move-to-child/newparent/dir3
+  dest-move-to-child/newparent/dir3/subdir5
+  dest-move-to-child/newparent/dir3/subdir5/subsubdir0
+  dest-move-to-child/newparent/dir3/subdir5/subsubdir0/file9
+  dest-move-to-child/newparent/file0
+
+Move multiple directories to different directories
+
+  $ mkdir dest-move-multiple
+  $ test-synchronize-directories source0 dest-move-multiple --map dir0:dir0-rewrite --map dir3:newparent/dir3-rewrite
+
+  $ find dest-move-multiple | sort
+  dest-move-multiple
+  dest-move-multiple/dir0-rewrite
+  dest-move-multiple/dir0-rewrite/file1
+  dest-move-multiple/dir0-rewrite/file2
+  dest-move-multiple/dir0-rewrite/subdir0
+  dest-move-multiple/dir0-rewrite/subdir0/file3
+  dest-move-multiple/dir0-rewrite/subdir1
+  dest-move-multiple/dir0-rewrite/subdir1/file4
+  dest-move-multiple/dir0-rewrite/subdir2
+  dest-move-multiple/dir0-rewrite/subdir2/file5
+  dest-move-multiple/dir0-rewrite/subdir2/file6
+  dest-move-multiple/newparent
+  dest-move-multiple/newparent/dir3-rewrite
+  dest-move-multiple/newparent/dir3-rewrite/subdir5
+  dest-move-multiple/newparent/dir3-rewrite/subdir5/subsubdir0
+  dest-move-multiple/newparent/dir3-rewrite/subdir5/subsubdir0/file9
+
+When using directory map, extra files in mapped directories should be removed.
+Extra files elsewhere should be preserved.
+
+  $ mkdir -p dest-map-extra-files/dir1 dest-map-extra-files/extra-dir
+  $ touch dest-map-extra-files/extra-file
+  $ touch dest-map-extra-files/extra-dir/extra-file
+  $ touch dest-map-extra-files/dir1/extra-file
+  $ test-synchronize-directories source0 dest-map-extra-files --map dir1:dir1
+
+  $ find dest-map-extra-files | sort
+  dest-map-extra-files
+  dest-map-extra-files/dir1
+  dest-map-extra-files/dir1/extra-file
+  dest-map-extra-files/dir1/subdir4
+  dest-map-extra-files/dir1/subdir4/file7
+  dest-map-extra-files/extra-dir
+  dest-map-extra-files/extra-dir/extra-file
+  dest-map-extra-files/extra-file