vcssync: support for converting a linearized Git repo to Mercurial (bug 1322769); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Thu, 05 Jan 2017 18:00:55 -0800
changeset 10223 eccdcb581828bfcea59f9affa3a1ec44f4cc4fd3
parent 10222 c7abbb778086bbf0ec13202850e7641c6a1db563
push id1480
push userbmo:gps@mozilla.com
push dateWed, 25 Jan 2017 19:14:03 +0000
reviewersglob
bugs1322769
vcssync: support for converting a linearized Git repo to Mercurial (bug 1322769); r?glob At Mozilla, we like monolithic repositories ("monorepos") where all the code lives in one repository. This has a number of advantages, especially around workflows and productivity. (We share this opinion with Google and Facebook among others.) At Mozilla, we also like open source. And a lot of open source happens in Git and on GitHub. Over the years, a number of groups at Mozilla have either started doing development work on Git[Hub] or have wanted to. This creates a problem for managing the Firefox source repository because a) it is canonically hosted in Mercurial b) it is a monorepo. The existence of Git-based development outside of mozilla-central fragments our code and workflows and creates an impedance mismatch between version control tools. While there may be local wins, overall it creates chaos. Our solution to this problem is to provide the best of both worlds: allow people to use Git[Hub] if they want while still providing all the benefits of the Mercurial-backed monorepo. This commit introduces code for converting a "linearized" Git repo to a Mercurial one. It builds on top of the previous commit implementing in-place Git history rewriting and adds `hg convert` to convert the rewritten Git history to Mercurial. The code is really pretty simple. Most of the lines deal with argument/parameter passing. The hard work is done elsewhere. The functionality as implemented is sufficient for a one-time or ongoing conversion of a Git repo to Mercurial. However, what it doesn't yet do (which will be required for ongoing vendoring of Servo commits), is "overlaying" the Git-based files onto the Mercurial repository. This isn't important for the "stage 1" import of Servo into mozilla-central so it will be implemented in another commit. MozReview-Commit-ID: HcF3LIrQYZV
vcssync/mozvcssync/cli.py
vcssync/mozvcssync/git2hg.py
vcssync/setup.py
vcssync/tests/helpers.sh
vcssync/tests/test-linearize-git-to-hg-arg-errors.t
vcssync/tests/test-linearize-git-to-hg-basic.t
vcssync/tests/test-linearize-git-to-hg-copy-metadata.t
vcssync/tests/test-linearize-git-to-hg-move-to-subdir.t
vcssync/tests/test-linearize-git-to-hg-push-git.t
vcssync/tests/test-linearize-git-to-hg-source-annotations.t
--- a/vcssync/mozvcssync/cli.py
+++ b/vcssync/mozvcssync/cli.py
@@ -1,18 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
 import argparse
 import logging
+import subprocess
 import sys
 
+import hglib
+
+from .git2hg import (
+    linearize_git_repo_to_hg,
+)
 from .gitrewrite import (
     RewriteError,
 )
 from .gitrewrite.linearize import (
     linearize_git_repo,
 )
 
 
@@ -105,8 +111,64 @@ def linearize_git():
 
     if args.source_repo:
         kwargs['source_repo'] = args.source_repo
 
     try:
         linearize_git_repo(args.git_repo, args.ref, **kwargs)
     except RewriteError as e:
         logger.error('abort: %s' % str(e))
+
+
+def linearize_git_to_hg():
+    parser = argparse.ArgumentParser()
+    for args, kwargs in LINEARIZE_GIT_ARGS:
+        parser.add_argument(*args, **kwargs)
+
+    parser.add_argument('--hg', help='hg executable to use')
+    parser.add_argument('--move-to-subdir',
+                        help='Move the files in the Git repository under a '
+                             'different subdirection in the destination repo')
+    parser.add_argument('--copy-similarity', type=int, default=50,
+                        dest='similarity',
+                        help='File % similarity for it to be identified as a '
+                             'copy')
+    parser.add_argument('--find-copies-harder', action='store_true',
+                        help='Work harder to find file copies')
+    parser.add_argument('--skip-submodules', action='store_true',
+                        help='Skip processing of Git submodules')
+    parser.add_argument('--git-push-url',
+                        help='URL where to push converted Git repo')
+    parser.add_argument('git_repo_url',
+                        help='URL of Git repository to convert')
+    parser.add_argument('git_ref', help='Git ref to convert')
+    parser.add_argument('git_repo_path', help='Local path of where to store '
+                                              'Git repo clone')
+    parser.add_argument('hg_repo_path', help='Local path of where to store '
+                                             'Mercurial conversion of repo')
+
+
+    args = parser.parse_args()
+
+    if args.hg:
+        hglib.HGPATH = args.hg
+
+    configure_logging()
+
+    kwargs = get_git_linearize_kwargs(args)
+    for k in ('similarity', 'find_copies_harder', 'skip_submodules',
+              'move_to_subdir', 'git_push_url'):
+        v = getattr(args, k)
+        if v is not None:
+            kwargs[k] = v
+
+    try:
+        linearize_git_repo_to_hg(
+            args.git_repo_url,
+            args.git_ref,
+            args.git_repo_path,
+            args.hg_repo_path,
+            **kwargs)
+    except RewriteError as e:
+        logger.error('abort: %s' % str(e))
+        sys.exit(1)
+    except subprocess.CalledProcessError:
+        sys.exit(1)
new file mode 100644
--- /dev/null
+++ b/vcssync/mozvcssync/git2hg.py
@@ -0,0 +1,204 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import subprocess
+import tempfile
+
+import hglib
+
+from .gitrewrite.linearize import (
+    linearize_git_repo,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def linearize_git_repo_to_hg(git_source_url, ref, git_repo, hg_repo,
+                             git_push_url=None,
+                             move_to_subdir=None,
+                             find_copies_harder=False,
+                             skip_submodules=False,
+                             similarity=50,
+                             **kwargs):
+    """Linearize a Git repo to an hg repo by squashing merges.
+
+    Many Git repositories (especially those on GitHub) have an excessive
+    number of merge commits and don't practice "every commit is
+    good/bisectable." When converting these repositories to Mercurial, it is
+    often desirable to ignore the non-first-parent ancestry so the result has
+    more readable history.
+
+    This function will perform such a conversion.
+
+    The source Git repository to convert is specified by ``git_source_url``
+    and ``ref``, where ``git_source_url`` is a URL understood by ``git
+    clone`` and ``ref`` is a Git ref, like ``master``. Only converting
+    a single ref is allowed.
+
+    The source Git repository is locally cloned to the path ``git_repo``.
+    This directory will be created if necessary.
+
+    If ``git_push_url`` is specified, the local clone (including converted
+    commits) will be pushed to that URL.
+
+    The conversion works in phases:
+
+    1) Git commits are rewritten into a new ref.
+    2) ``hg convert`` converts the rewritten Git commits to Mercurial.
+
+    See the docs in ``/docs/vcssync.rst`` for reasons why.
+
+    Returns a dict describing the conversion result. The dict has the following
+    keys:
+
+    git_result
+       This is a dict from ``linearize_git_repo()`` describing its results.
+    rev_map_path
+       Filesystem path to file mapping Git commit to Mercurial commit.
+    hg_before_tip_rev
+       Numeric revision of ``tip`` Mercurial changeset before conversion. ``-1``
+       if the repo was empty.
+    hg_before_tip_node
+       SHA-1 of ``tip`` Mercurial changeset before conversion. 40 0's if the
+       repo was empty.
+    hg_after_tip_rev
+       Numeric revision of ``tip`` Mercurial changeset before conversion.
+    hg_after_tip_node
+       SHA-1 of ``tip`` Mercurial changeset after conversion.
+    """
+    # Many processes execute with cwd=/ so normalize to absolute paths.
+    git_repo = os.path.abspath(git_repo)
+    hg_repo = os.path.abspath(hg_repo)
+
+    # Create Git repo, if necessary.
+    if not os.path.exists(git_repo):
+        subprocess.check_call([b'git', b'init', b'--bare', git_repo])
+        # We don't need to set up a remote because we use an explicit refspec
+        # during fetch.
+
+    subprocess.check_call([b'git', b'fetch', b'--no-tags', git_source_url,
+                           b'heads/%s:heads/%s' % (ref, ref)],
+                          cwd=git_repo)
+
+    if git_push_url:
+        subprocess.check_call([b'git', b'push', b'--mirror', git_push_url],
+                              cwd=git_repo)
+
+    git_state = linearize_git_repo(
+        git_repo,
+        b'heads/%s' % ref,
+        source_repo=git_source_url,
+        **kwargs)
+
+    if git_push_url:
+        subprocess.check_call([b'git', b'push', b'--mirror', git_push_url],
+                              cwd=git_repo)
+
+    rev_map = os.path.join(hg_repo, b'.hg', b'shamap')
+
+    result = {
+        'git_result': git_state,
+        'rev_map_path': rev_map,
+    }
+
+    # If nothing was converted, no-op if the head is already converted
+    # according to the `hg convert` revision map.
+    if not git_state['commit_map']:
+        try:
+            with open(rev_map, 'rb') as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    shas = line.split()
+                    if shas[0] == git_state['dest_commit']:
+                        logger.warn('all Git commits have already been '
+                                    'converted; not doing anything')
+                        return result
+        except IOError:
+            # Fall through to doing the conversion. If it's a file permissions
+            # error, `hg convert` will abort.
+            pass
+
+    logger.warn('converting %d Git commits' % len(git_state['commit_map']))
+
+    hg_config = [
+        b'extensions.convert=',
+        # Make the rename detection limit essentially infinite.
+        b'convert.git.renamelimit=1000000000',
+        # The ``convert_revision`` that would be stored reflects the rewritten
+        # Git commit. This is valuable as a persistent SHA map, but that's it.
+        # We (hopefully) insert the original Git commit via
+        # ``source_revision_key``, so this is of marginal value.
+        b'convert.git.saverev=false',
+        b'convert.git.similarity=%d' % similarity,
+    ]
+
+    if find_copies_harder:
+        hg_config.append(b'convert.git.findcopiesharder=true')
+    if skip_submodules:
+        hg_config.append(b'convert.git.skipsubmodules=true')
+
+    if not os.path.exists(hg_repo):
+        hglib.init(hg_repo)
+
+    with hglib.open(hg_repo) as hrepo:
+        tip = hrepo[b'tip']
+        before_hg_tip_rev = tip.rev()
+        before_hg_tip_node = tip.node()
+
+    args = [hglib.HGPATH]
+    for c in hg_config:
+        args.extend([b'--config', c])
+
+    args.extend([b'convert'])
+    args.extend([b'--rev', b'refs/convert/dest/heads/%s' % ref])
+
+    # `hg convert` needs a filemap to prune empty changesets. So use an
+    # empty file even if we don't have any filemap rules.
+    with tempfile.NamedTemporaryFile('wb') as tf:
+        if move_to_subdir:
+            tf.write(b'rename . %s\n' % move_to_subdir)
+
+        tf.flush()
+
+        args.extend([b'--filemap', tf.name])
+
+        args.extend([git_repo, hg_repo, rev_map])
+
+        # hglib doesn't appear to stream output very well. So just invoke
+        # `hg` directly.
+        env = dict(os.environ)
+        env[b'HGPLAIN'] = b'1'
+        env[b'HGENCODING'] = b'utf-8'
+
+        subprocess.check_call(args, cwd='/', env=env)
+
+    with hglib.open(hg_repo) as hrepo:
+        tip = hrepo[b'tip']
+        after_hg_tip_rev = tip.rev()
+        after_hg_tip_node = tip.node()
+
+    if before_hg_tip_rev == -1:
+        convert_count = after_hg_tip_rev + 1
+    else:
+        convert_count = after_hg_tip_rev - before_hg_tip_rev
+
+    result['hg_before_tip_rev'] = before_hg_tip_rev
+    result['hg_after_tip_rev'] = after_hg_tip_rev
+    result['hg_before_tip_node'] = before_hg_tip_node
+    result['hg_after_tip_node'] = after_hg_tip_node
+    result['hg_convert_count'] = convert_count
+
+    logger.warn('%d Git commits converted to Mercurial; '
+                'previous tip: %d:%s; current tip: %d:%s' % (
+        convert_count, before_hg_tip_rev, before_hg_tip_node,
+        after_hg_tip_rev, after_hg_tip_node))
+
+    return result
--- a/vcssync/setup.py
+++ b/vcssync/setup.py
@@ -12,12 +12,13 @@ setup(
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
         'Programming Language :: Python :: 2.7',
     ],
     packages=find_packages(),
     entry_points={
         'console_scripts': [
             'linearize-git=mozvcssync.cli:linearize_git',
+            'linearize-git-to-hg=mozvcssync.cli:linearize_git_to_hg',
         ],
     },
     install_requires=['dulwich>=0.16', 'github3.py>=0.9.6', 'Mercurial>=4.0'],
 )
--- a/vcssync/tests/helpers.sh
+++ b/vcssync/tests/helpers.sh
@@ -6,8 +6,110 @@
 
 # make git commits deterministic and environment agnostic
 export GIT_AUTHOR_NAME=test
 export GIT_AUTHOR_EMAIL=test@example.com
 export GIT_AUTHOR_DATE='Thu Jan 1 00:00:00 1970 +0000'
 export GIT_COMMITTER_NAME=test
 export GIT_COMMITTER_EMAIL=test@example.com
 export GIT_COMMITTER_DATE='Thu Jan 1 00:00:00 1970 +0000'
+
+standardgitrepo() {
+    here=`pwd`
+    git init $1
+    cd $1
+    echo 0 > foo
+    git add foo
+    git commit -m initial
+    cat > file0 << EOF
+file0 0
+file0 1
+file0 2
+file0 3
+file0 4
+file0 5
+file0 6
+file0 7
+file0 8
+file0 9
+file0 10
+EOF
+    cat > file1 << EOF
+file1 0
+file1 1
+file1 2
+file1 3
+file1 4
+file1 5
+file1 6
+file1 7
+file1 8
+file1 9
+EOF
+
+    git add file0 file1
+    git commit -m 'add file0 and file1'
+    cp file0 file0-copy0
+    git add file0-copy0
+    git commit -m 'copy file0 to file0-copy0'
+    cp file0 file0-copy1
+    cp file0 file0-copy2
+    git add file0-copy1 file0-copy2
+    git commit -m 'copy file0 to file0-copy1 and file0-copy2'
+    git mv file0 file0-moved
+    git commit -m 'move file0 to file0-moved'
+
+    # Make copy then move source so default copy detection kicks in
+    cp file0-moved file0-copied-with-move
+    git mv file0-moved file0-moved-with-copy
+    git add file0-copied-with-move
+    git commit -m 'copy file0-moved and rename source'
+
+    # Create copies of file1 with modifications
+    cat > file1-20 << EOF
+file1 2
+file1 7
+EOF
+
+    cat > file1-50 << EOF
+file1 0
+file1 1
+file1 2
+file1 3
+file1 4
+EOF
+
+   cat > file1-80 << EOF
+file1 0
+file1 1
+file1 2
+file1 3
+file1 5
+file1 6
+file1 7
+file1 9
+EOF
+
+    git add file1-20 file1-50 file1-80
+    git commit -m 'create file1-20, file1-50 and file1-80 as copies with mods'
+
+    git branch head2
+    echo 1 > foo
+    git add foo
+    git commit -m 'dummy commit 1 on master'
+    echo 2 > foo
+    git add foo
+    git commit -m 'dummy commit 2 on master'
+    git checkout head2
+    echo 3 > bar
+    git add bar
+    git commit -m 'dummy commit 1 on head2'
+    echo 4 > bar
+    git add bar
+    git commit -m 'dummy commit 2 on head2'
+    git checkout master
+    git merge head2
+    echo 5 > foo
+    git add foo
+    git commit -m 'dummy commit 1 after merge'
+
+    cd $here
+}
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-arg-errors.t
@@ -0,0 +1,21 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+Git source URL does not exist
+
+  $ linearize-git-to-hg file://$TESTTMP/does/not/exist master dummy dummy
+  Initialized empty Git repository in $TESTTMP/dummy/
+  fatal: '$TESTTMP/does/not/exist' does not appear to be a git repository
+  fatal: Could not read from remote repository.
+  
+  Please make sure you have the correct access rights
+  and the repository exists.
+  [1]
+
+Source ref does not exist
+
+  $ linearize-git-to-hg file://$TESTTMP/grepo badref grepo-clone dummy
+  Initialized empty Git repository in $TESTTMP/grepo-clone/
+  fatal: Couldn't find remote ref heads/badref
+  fatal: The remote end hung up unexpectedly
+  [1]
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-basic.t
@@ -0,0 +1,62 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+  $ git --git-dir grepo/.git log --graph --format=oneline
+  * a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+  *   fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+  |\  
+  | * 7b4eab003357aaa2873a1976cb86f5c4a70f5f22 dummy commit 2 on head2
+  | * 8b358b77dede1a070e1047874be3679a31a210c9 dummy commit 1 on head2
+  * | 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+  * | 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+  |/  
+  * ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+  * 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+  * 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+  * 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+  * 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+  * 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+  * dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+
+Basic git to hg linearize works
+
+  $ linearize-git-to-hg file://$TESTTMP/grepo master grepo-source grepo-dest
+  Initialized empty Git repository in $TESTTMP/grepo-source/
+  From file://$TESTTMP/grepo
+   * [new branch]      master     -> master
+  linearizing 11 commits from heads/master (dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf to a447b9b0ff25bf17daab1c7edae4a998eca0adac)
+  1/11 dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+  2/11 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+  3/11 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+  4/11 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+  5/11 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+  6/11 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+  7/11 ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+  8/11 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+  9/11 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+  10/11 fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+  11/11 a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+  heads/master converted; original: a447b9b0ff25bf17daab1c7edae4a998eca0adac; rewritten: aea30981234cf6848489e0ccf541fbf902b27aca
+  converting 11 Git commits
+  scanning source...
+  sorting...
+  converting...
+  10 initial
+  9 add file0 and file1
+  8 copy file0 to file0-copy0
+  7 copy file0 to file0-copy1 and file0-copy2
+  6 move file0 to file0-moved
+  5 copy file0-moved and rename source
+  4 create file1-20, file1-50 and file1-80 as copies with mods
+  3 dummy commit 1 on master
+  2 dummy commit 2 on master
+  1 Merge branch 'head2'
+  0 dummy commit 1 after merge
+  11 Git commits converted to Mercurial; previous tip: -1:0000000000000000000000000000000000000000; current tip: 10:74b93af557b18fa56b0e9fad513ef9da1a1d950f
+
+Subsequent invocation no-ops
+
+  $ linearize-git-to-hg file://$TESTTMP/grepo master grepo-source grepo-dest
+  no new commits to linearize; not doing anything
+  all Git commits have already been converted; not doing anything
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-copy-metadata.t
@@ -0,0 +1,214 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+
+  $ cat >> $HGRCPATH << EOF
+  > [diff]
+  > git = true
+  > EOF
+
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+  $ linearize-git-to-hg file://$TESTTMP/grepo master grepo-source grepo-dest-default
+  Initialized empty Git repository in $TESTTMP/grepo-source/
+  From file://$TESTTMP/grepo
+   * [new branch]      master     -> master
+  linearizing 11 commits from heads/master (dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf to a447b9b0ff25bf17daab1c7edae4a998eca0adac)
+  1/11 dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+  2/11 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+  3/11 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+  4/11 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+  5/11 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+  6/11 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+  7/11 ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+  8/11 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+  9/11 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+  10/11 fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+  11/11 a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+  heads/master converted; original: a447b9b0ff25bf17daab1c7edae4a998eca0adac; rewritten: aea30981234cf6848489e0ccf541fbf902b27aca
+  converting 11 Git commits
+  scanning source...
+  sorting...
+  converting...
+  10 initial
+  9 add file0 and file1
+  8 copy file0 to file0-copy0
+  7 copy file0 to file0-copy1 and file0-copy2
+  6 move file0 to file0-moved
+  5 copy file0-moved and rename source
+  4 create file1-20, file1-50 and file1-80 as copies with mods
+  3 dummy commit 1 on master
+  2 dummy commit 2 on master
+  1 Merge branch 'head2'
+  0 dummy commit 1 after merge
+  11 Git commits converted to Mercurial; previous tip: -1:0000000000000000000000000000000000000000; current tip: 10:74b93af557b18fa56b0e9fad513ef9da1a1d950f
+
+Move annotation should be preserved automatically
+
+  $ hg -R grepo-dest-default export 4
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID 045473c2b7065422c50de2de883bbfabd42307e9
+  # Parent  4670490276d96d9f5aafbcfba095b94401ce2f7b
+  move file0 to file0-moved
+  
+  diff --git a/file0 b/file0-moved
+  rename from file0
+  rename to file0-moved
+
+Copy annotation should be preserved automatically
+
+  $ hg -R grepo-dest-default export 5
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID 2f1c561c2bba03de041ae91b9bd996a8e1592a46
+  # Parent  045473c2b7065422c50de2de883bbfabd42307e9
+  copy file0-moved and rename source
+  
+  diff --git a/file0-moved b/file0-copied-with-move
+  rename from file0-moved
+  rename to file0-copied-with-move
+  diff --git a/file0-moved b/file0-moved-with-copy
+  copy from file0-moved
+  copy to file0-moved-with-copy
+
+Normal copy won't be detected if source not modified
+
+  $ hg -R grepo-dest-default export 2
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID ef5cf78e1c5224adea8f6cebba99ed88bed64389
+  # Parent  9b56156dddc3d86f3afaba790ad91260c85c74b2
+  copy file0 to file0-copy0
+  
+  diff --git a/file0-copy0 b/file0-copy0
+  new file mode 100644
+  --- /dev/null
+  +++ b/file0-copy0
+  @@ -0,0 +1,11 @@
+  +file0 0
+  +file0 1
+  +file0 2
+  +file0 3
+  +file0 4
+  +file0 5
+  +file0 6
+  +file0 7
+  +file0 8
+  +file0 9
+  +file0 10
+
+--find-copies-harder will find copies
+
+  $ linearize-git-to-hg --find-copies-harder file://$TESTTMP/grepo master grepo-source grepo-dest-find-copy-harder > /dev/null 2>&1
+
+  $ hg -R grepo-dest-find-copy-harder export 2
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID 69b9ae19ba84e5c77b119294715e992a8d7070bf
+  # Parent  9b56156dddc3d86f3afaba790ad91260c85c74b2
+  copy file0 to file0-copy0
+  
+  diff --git a/file0 b/file0-copy0
+  copy from file0
+  copy to file0-copy0
+
+Copy detection similarity is sane
+
+  $ hg -R grepo-dest-find-copy-harder export 6
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID b5b3e192ecdecbfc25d1ef6241664db3c63ef4d0
+  # Parent  c1afa6c62b584e017c1a3900055d2eba740092b5
+  create file1-20, file1-50 and file1-80 as copies with mods
+  
+  diff --git a/file1-20 b/file1-20
+  new file mode 100644
+  --- /dev/null
+  +++ b/file1-20
+  @@ -0,0 +1,2 @@
+  +file1 2
+  +file1 7
+  diff --git a/file1 b/file1-50
+  copy from file1
+  copy to file1-50
+  --- a/file1
+  +++ b/file1-50
+  @@ -3,8 +3,3 @@
+   file1 2
+   file1 3
+   file1 4
+  -file1 5
+  -file1 6
+  -file1 7
+  -file1 8
+  -file1 9
+  diff --git a/file1 b/file1-80
+  copy from file1
+  copy to file1-80
+  --- a/file1
+  +++ b/file1-80
+  @@ -2,9 +2,7 @@
+   file1 1
+   file1 2
+   file1 3
+  -file1 4
+   file1 5
+   file1 6
+   file1 7
+  -file1 8
+   file1 9
+
+Increase similarity threshold removes copy annotation from file1-50
+
+  $ linearize-git-to-hg --copy-similarity 70 --find-copies-harder file://$TESTTMP/grepo master grepo-source grepo-dest-similarity-70 > /dev/null 2>&1
+
+  $ hg -R grepo-dest-similarity-70 export 6
+  # HG changeset patch
+  # User test <test@example.com>
+  # Date 0 0
+  #      Thu Jan 01 00:00:00 1970 +0000
+  # Node ID 2cbb8f45b28f77c4b45382629a4b9fa297df0960
+  # Parent  c1afa6c62b584e017c1a3900055d2eba740092b5
+  create file1-20, file1-50 and file1-80 as copies with mods
+  
+  diff --git a/file1-20 b/file1-20
+  new file mode 100644
+  --- /dev/null
+  +++ b/file1-20
+  @@ -0,0 +1,2 @@
+  +file1 2
+  +file1 7
+  diff --git a/file1-50 b/file1-50
+  new file mode 100644
+  --- /dev/null
+  +++ b/file1-50
+  @@ -0,0 +1,5 @@
+  +file1 0
+  +file1 1
+  +file1 2
+  +file1 3
+  +file1 4
+  diff --git a/file1 b/file1-80
+  copy from file1
+  copy to file1-80
+  --- a/file1
+  +++ b/file1-80
+  @@ -2,9 +2,7 @@
+   file1 1
+   file1 2
+   file1 3
+  -file1 4
+   file1 5
+   file1 6
+   file1 7
+  -file1 8
+   file1 9
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-move-to-subdir.t
@@ -0,0 +1,36 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+--move-to-subdir will move files in git repo to subdirectory in hg
+
+  $ linearize-git-to-hg --move-to-subdir subdir file://$TESTTMP/grepo master grepo-source grepo-dest-0 > /dev/null 2>&1
+
+  $ hg --cwd grepo-dest-0 files -r tip
+  subdir/bar
+  subdir/file0-copied-with-move
+  subdir/file0-copy0
+  subdir/file0-copy1
+  subdir/file0-copy2
+  subdir/file0-moved-with-copy
+  subdir/file1
+  subdir/file1-20
+  subdir/file1-50
+  subdir/file1-80
+  subdir/foo
+
+Multiple child directories works
+
+  $ linearize-git-to-hg --move-to-subdir dir0/dir1/dir2 file://$TESTTMP/grepo master grepo-source grepo-dest-1 > /dev/null 2>&1
+
+  $ hg --cwd grepo-dest-1 files -r tip
+  dir0/dir1/dir2/bar
+  dir0/dir1/dir2/file0-copied-with-move
+  dir0/dir1/dir2/file0-copy0
+  dir0/dir1/dir2/file0-copy1
+  dir0/dir1/dir2/file0-copy2
+  dir0/dir1/dir2/file0-moved-with-copy
+  dir0/dir1/dir2/file1
+  dir0/dir1/dir2/file1-20
+  dir0/dir1/dir2/file1-50
+  dir0/dir1/dir2/file1-80
+  dir0/dir1/dir2/foo
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-push-git.t
@@ -0,0 +1,98 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+  $ git init --bare grepo-mirror
+  Initialized empty Git repository in $TESTTMP/grepo-mirror/
+
+--git-push-url will mirror the local Git repo to a remote after changes
+
+  $ linearize-git-to-hg --git-push-url file://$TESTTMP/grepo-mirror file://$TESTTMP/grepo master grepo-source hgrepo-dest
+  Initialized empty Git repository in $TESTTMP/grepo-source/
+  From file://$TESTTMP/grepo
+   * [new branch]      master     -> master
+  To file://$TESTTMP/grepo-mirror
+   * [new branch]      master -> master
+  linearizing 11 commits from heads/master (dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf to a447b9b0ff25bf17daab1c7edae4a998eca0adac)
+  1/11 dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+  2/11 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+  3/11 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+  4/11 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+  5/11 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+  6/11 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+  7/11 ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+  8/11 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+  9/11 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+  10/11 fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+  11/11 a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+  heads/master converted; original: a447b9b0ff25bf17daab1c7edae4a998eca0adac; rewritten: aea30981234cf6848489e0ccf541fbf902b27aca
+  To file://$TESTTMP/grepo-mirror
+   * [new branch]      refs/convert/dest/heads/master -> refs/convert/dest/heads/master
+   * [new branch]      refs/convert/source/heads/master -> refs/convert/source/heads/master
+  converting 11 Git commits
+  scanning source...
+  sorting...
+  converting...
+  10 initial
+  9 add file0 and file1
+  8 copy file0 to file0-copy0
+  7 copy file0 to file0-copy1 and file0-copy2
+  6 move file0 to file0-moved
+  5 copy file0-moved and rename source
+  4 create file1-20, file1-50 and file1-80 as copies with mods
+  3 dummy commit 1 on master
+  2 dummy commit 2 on master
+  1 Merge branch 'head2'
+  0 dummy commit 1 after merge
+  11 Git commits converted to Mercurial; previous tip: -1:0000000000000000000000000000000000000000; current tip: 10:74b93af557b18fa56b0e9fad513ef9da1a1d950f
+
+All refs from local Git repo should be in mirror
+
+  $ git -C grepo-source for-each-ref
+  aea30981234cf6848489e0ccf541fbf902b27aca commit	refs/convert/dest/heads/master
+  a447b9b0ff25bf17daab1c7edae4a998eca0adac commit	refs/convert/source/heads/master
+  a447b9b0ff25bf17daab1c7edae4a998eca0adac commit	refs/heads/master
+
+  $ git -C grepo-mirror for-each-ref
+  aea30981234cf6848489e0ccf541fbf902b27aca commit	refs/convert/dest/heads/master
+  a447b9b0ff25bf17daab1c7edae4a998eca0adac commit	refs/convert/source/heads/master
+  a447b9b0ff25bf17daab1c7edae4a998eca0adac commit	refs/heads/master
+
+Incremental conversion will keep Git repo in sync
+
+  $ cd grepo
+  $ touch incremental
+  $ git add incremental
+  $ git commit -m 'add incremental'
+  [master 4040c16] add incremental
+   1 file changed, 0 insertions(+), 0 deletions(-)
+   create mode 100644 incremental
+  $ cd ..
+
+  $ linearize-git-to-hg --git-push-url file://$TESTTMP/grepo-mirror file://$TESTTMP/grepo master grepo-source hgrepo-dest
+  From file://$TESTTMP/grepo
+     a447b9b..4040c16  master     -> master
+  To file://$TESTTMP/grepo-mirror
+     a447b9b..4040c16  master -> master
+  linearizing 1 commits from heads/master (4040c1631489c25dd4e0fd1606c4a065e1a24194 to 4040c1631489c25dd4e0fd1606c4a065e1a24194)
+  1/1 4040c1631489c25dd4e0fd1606c4a065e1a24194 add incremental
+  heads/master converted; original: 4040c1631489c25dd4e0fd1606c4a065e1a24194; rewritten: d6ec61184bff36a58159341c2584f3cda9dd0b58
+  To file://$TESTTMP/grepo-mirror
+     aea3098..d6ec611  refs/convert/dest/heads/master -> refs/convert/dest/heads/master
+     a447b9b..4040c16  refs/convert/source/heads/master -> refs/convert/source/heads/master
+  converting 1 Git commits
+  scanning source...
+  sorting...
+  converting...
+  0 add incremental
+  1 Git commits converted to Mercurial; previous tip: 10:74b93af557b18fa56b0e9fad513ef9da1a1d950f; current tip: 11:b53d6fba975e3face586964aace142716b2191a7
+
+  $ git -C grepo-source for-each-ref
+  d6ec61184bff36a58159341c2584f3cda9dd0b58 commit	refs/convert/dest/heads/master
+  4040c1631489c25dd4e0fd1606c4a065e1a24194 commit	refs/convert/source/heads/master
+  4040c1631489c25dd4e0fd1606c4a065e1a24194 commit	refs/heads/master
+
+  $ git -C grepo-mirror for-each-ref
+  d6ec61184bff36a58159341c2584f3cda9dd0b58 commit	refs/convert/dest/heads/master
+  4040c1631489c25dd4e0fd1606c4a065e1a24194 commit	refs/convert/source/heads/master
+  4040c1631489c25dd4e0fd1606c4a065e1a24194 commit	refs/heads/master
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-linearize-git-to-hg-source-annotations.t
@@ -0,0 +1,61 @@
+  $ . $TESTDIR/vcssync/tests/helpers.sh
+  $ standardgitrepo grepo > /dev/null 2>&1
+
+Note: since source repo is in $TESTTMP which is dynamic, commit SHA-1s aren't stable!
+
+  $ linearize-git-to-hg --source-repo-key Source-Repo --source-revision-key Source-Revision file://$TESTTMP/grepo master grepo-source grepo-dest
+  Initialized empty Git repository in $TESTTMP/grepo-source/
+  From file://$TESTTMP/grepo
+   * [new branch]      master     -> master
+  linearizing 11 commits from heads/master (dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf to a447b9b0ff25bf17daab1c7edae4a998eca0adac)
+  1/11 dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+  2/11 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+  3/11 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+  4/11 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+  5/11 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+  6/11 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+  7/11 ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+  8/11 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+  9/11 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+  10/11 fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+  11/11 a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+  heads/master converted; original: a447b9b0ff25bf17daab1c7edae4a998eca0adac; rewritten: * (glob)
+  converting 11 Git commits
+  scanning source...
+  sorting...
+  converting...
+  10 initial
+  9 add file0 and file1
+  8 copy file0 to file0-copy0
+  7 copy file0 to file0-copy1 and file0-copy2
+  6 move file0 to file0-moved
+  5 copy file0-moved and rename source
+  4 create file1-20, file1-50 and file1-80 as copies with mods
+  3 dummy commit 1 on master
+  2 dummy commit 2 on master
+  1 Merge branch 'head2'
+  0 dummy commit 1 after merge
+  11 Git commits converted to Mercurial; previous tip: -1:0000000000000000000000000000000000000000; current tip: 10:* (glob)
+
+Key: Value metadata identifying source should appear in commit message
+TODO Mercurial 4.1 will remove convert_revision from extra, which is wanted
+
+  $ hg -R grepo-dest log --debug -r 0
+  changeset:   0:* (glob)
+  phase:       draft
+  parent:      -1:0000000000000000000000000000000000000000
+  parent:      -1:0000000000000000000000000000000000000000
+  manifest:    0:cba485ca3678256e044428f70f58291196f6e9de
+  user:        test <test@example.com>
+  date:        Thu Jan 01 00:00:00 1970 +0000
+  files+:      foo
+  extra:       branch=default
+  extra:       convert_revision=* (glob)
+  description:
+  initial
+  
+  Source-Repo: file://$TESTTMP/grepo
+  Source-Revision: dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf
+  
+  
+