vcssync: support for squashing a Git ref into a Mercurial repo (
bug 1357597); r?glob
It is common to want to incorporate changes in a Git branch into
another repository. This is commonly done via a `git merge`. If using
GitHub, you may refer to this as "merging a pull request."
The core of the operation involves taking 2 Git commits (often defined
by refs) and merging them. This produces a tree of files with the
new content.
The code in this commit provides a generic facility for obtaining
the result of a `git merge` and importing it into a Mercurial
repository. It does this by doing a `git merge` then synchronizing
working directories. It does things in such a way that results
should be deterministic and should honor VCS semantics (such as
ignoring files that should be ignored). The implementation also
supports remapping directories between the working directories.
This facilitates common scenarios like modeling a Git repo as
a subdirectory as a monorepo.
The function is implemented as a generator that yields control to
the caller at key phases. This allows callers to have almost full
control over what is done when. One caller could dynamically pick
the destination revision in the Mercurial repository based on the
Git base revision. Another caller may want to create a Mercurial
commit that "synchronizes" the Mercurial repo with the Git base
revision.
Test coverage is a bit light. I'd rather wait for a practical consumer
(like a GitHub PR to Try converter) before we go too hard on testing.
A more complex variation of this code would preserve each commit
in the Git head branch. Implementing that is for another day.
MozReview-Commit-ID: EcCFpGWlvo7
--- a/vcssync/mozvcssync/cli.py
+++ b/vcssync/mozvcssync/cli.py
@@ -4,24 +4,26 @@
from __future__ import absolute_import, unicode_literals
import argparse
import logging
import os
import subprocess
import sys
+import tempfile
import dulwich.repo
import hglib
import pathlib2 as pathlib
from .git2hg import (
copy_git_workdir_to_hg_workdir,
linearize_git_repo_to_hg,
+ overlay_git_ref_via_squash,
)
from .gitrewrite import (
RewriteError,
commit_metadata_rewriter,
)
from .gitrewrite.linearize import (
linearize_git_repo,
)
@@ -29,16 +31,17 @@ from .overlay import (
overlay_hg_repos,
PushRaceError,
PushRemoteFail,
)
from .servo_backout import (
backout_servo_pr,
)
from .util import (
+ run_hg,
synchronize_directories,
)
logger = logging.getLogger(__name__)
LINEARIZE_GIT_ARGS = [
(('--exclude-dir',), dict(action='append', dest='exclude_dirs',
@@ -375,8 +378,82 @@ def test_copy_workdir_git_to_hg():
k, v = e.split(':')
dir_map[k] = v
try:
copy_git_workdir_to_hg_workdir(args.source, args.dest, dir_map)
except Exception as e:
print('%s: %s' % (e.__class__.__name__, e))
sys.exit(1)
+
+
+def test_overlay_git_ref_via_squash():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('git_base_url',
+ help='URL of Git repo where base commit lives')
+ parser.add_argument('git_base_ref',
+ help='Ref in Git repo to intograte to')
+ parser.add_argument('git_head_url',
+ help='URL of Git repo where head commit lives')
+ parser.add_argument('git_head_ref',
+ help='Ref in Git repo to integrate from')
+ parser.add_argument('hg_repo_url',
+ help='URL of Mercurial repo where changes should be '
+ 'overlayed')
+ parser.add_argument('hg_dest_rev',
+ help='Revision in destination Mercurial repo where '
+ 'changes should be overlayed')
+ parser.add_argument('git_repo',
+ help='Path to local Git repo')
+ parser.add_argument('hg_repo',
+ help='Path to local Mercurial repo')
+ parser.add_argument('--map', action='append',
+ help='source:dest directory mapping')
+
+ args = parser.parse_args()
+ configure_logging()
+
+ dir_map = {}
+ for e in args.map or []:
+ k, v = e.split(':')
+ dir_map[k] = v
+
+ # This is needed to ensure temp directories are under test harness.
+ if b'TESTTMP' in os.environ:
+ tempfile.tempdir = os.path.join(os.environ[b'TESTTMP'], b'tmpdir')
+
+ commit_args = {
+ 'date': os.environ.get('HGDATE', None),
+ }
+
+ it = overlay_git_ref_via_squash(
+ args.git_base_url, args.git_base_ref,
+ args.git_head_url, args.git_head_ref,
+ git_repo_path=args.git_repo,
+ hg_repo_path=args.hg_repo,
+ dir_map=dir_map)
+
+ state = next(it)
+
+ run_hg(logger, state['hg_repo'],
+ [b'pull', b'-r', args.hg_dest_rev, args.hg_repo_url])
+ run_hg(logger, state['hg_repo'], [b'update', args.hg_dest_rev])
+
+ state, commit_needed = next(it)
+
+ print('git base commit: %s' % state['git_base_commit'].id)
+ print('git head commit: %s' % state['git_head_commit'].id)
+
+ if commit_needed:
+ logger.warn('committing in Mercurial to synchronize repo with '
+ 'Git base revision')
+ with hglib.open(args.hg_repo, encoding='utf-8') as hrepo:
+ synchronize_commit = hrepo.commit(
+ message='synchronize base', **commit_args)[1]
+ logger.warn('committed %s' % synchronize_commit)
+
+ state, commit_needed = next(it)
+ logger.warn('committing in Mercurial to synchronize repo with Git '
+ 'head revision')
+ with hglib.open(args.hg_repo, encoding='utf-8') as hrepo:
+ integrate_commit = hrepo.commit(
+ message='integrate head', **commit_args)[1]
+ logger.warn('committed %s' % integrate_commit)
--- a/vcssync/mozvcssync/git2hg.py
+++ b/vcssync/mozvcssync/git2hg.py
@@ -4,27 +4,34 @@
from __future__ import absolute_import, unicode_literals
import errno
import logging
import os
import subprocess
import tempfile
+import uuid
import dulwich.repo
import hglib
from .gitrewrite import (
commit_metadata_rewriter,
)
from .gitrewrite.linearize import (
linearize_git_repo,
)
+from .gitutil import (
+ ensure_revision_present,
+ temporary_git_checkout,
+ temporary_git_refs_mutations,
+)
from .util import (
+ clean_hg_repo,
monitor_hg_repo,
synchronize_directories,
)
logger = logging.getLogger(__name__)
@@ -324,8 +331,126 @@ def copy_git_workdir_to_hg_workdir(git_r
with hglib.open(hg_repo_path, encoding='utf-8', configs=configs) as hrepo:
hrepo.addremove()
for code, path in hrepo.status():
if code in (b'M', b'A', b'R'):
have_changes = True
break
return have_changes
+
+
+def overlay_git_ref_via_squash(
+ git_base_url, git_base_ref,
+ git_head_url, git_head_ref,
+ git_repo_path,
+ hg_repo_path,
+ dir_map=None,
+ git_base_revision=None,
+ git_head_revision=None):
+ """Overlay changes from a Git ref into a Mercurial repo with a squash
+ commit.
+
+ This function was invented to turn Git integration requests (such as GitHub
+ pull requests) into Mercurial changesets by squashing the result of the
+ "merge".
+
+ This function operates as a generator where each ``yield`` corresponds
+ to a specific phase in the overlay process. The function executes thusly:
+
+ 1. Git refs defined by ``git_base_url`` + ``git_base_ref`` and
+ ``git_head_url`` + ``git_head_ref`` are fetched into the local Git
+ repo at ``git_repo_path``.
+ 2. A state dict is yielded to the caller. This yield allows the caller
+ to inspect the Git and Mercurial repos and perform any additional
+ action, such as update the Mercurial repository to a specific revision.
+ 3. Files from ``git_base_ref`` are overlayed into the Mercurial repo using
+ the settings defined by ``dir_map``.
+ 4. A state dict and a boolean indicating if the Mercurial working directory
+ is dirty are yielded to the caller. This gives callers the opportunity
+ to inspect the repo or even perform a commit.
+ 5. A ``git merge`` of ``git_head_ref`` into ``git_base_ref`` is performed
+ and the Git working directory is synchronized to Mercurial's.
+ 6. A state dict is yielded to the caller. This allows the caller to inspect
+ the Mercurial repository and perform a commit, if wanted.
+
+ By yielding control at specific times, the function allow callers to perform
+ specific behavior (or even abort the procedure) as they see fit. This allows
+ the function to be generic and reusable in different contexts.
+
+ The passed ``git_repo_path`` may have its refs mutated during execution.
+ However, the old refs will be restored when the function returns. The
+ working directory is not modified. Instead, a temporary Git repository is
+ made to perform working directory operations.
+ """
+ instance_id = str(uuid.uuid4())
+
+ git_repo_path = os.path.abspath(git_repo_path)
+
+ local_base_ref = b'refs/overlay/%s/base' % instance_id
+ local_head_ref = b'refs/overlay/%s/head' % instance_id
+
+ git_repo = dulwich.repo.Repo(git_repo_path)
+
+ if not os.path.exists(hg_repo_path):
+ hglib.init(hg_repo_path)
+
+ # TODO consider creating the temp repo first and storing refs there so
+ # they don't contaminate original repo (if that's even possible).
+ with temporary_git_refs_mutations(git_repo), \
+ hglib.open(hg_repo_path, encoding='utf-8') as hg_repo:
+ ensure_revision_present(git_repo, git_base_revision, git_base_url,
+ git_base_ref, local_base_ref)
+ ensure_revision_present(git_repo, git_head_revision, git_head_url,
+ git_head_ref, local_head_ref)
+
+ base_commit = git_repo[git_base_revision or local_base_ref]
+ head_commit = git_repo[git_head_revision or local_head_ref]
+
+ state = {
+ 'git_base_commit': base_commit,
+ 'git_head_commit': head_commit,
+ 'git_repo': git_repo,
+ 'hg_repo': hg_repo,
+ }
+
+ yield state
+
+ # Ensure the Mercurial working directory is clean before any copies are
+ # performed.
+ # TODO consider having caller control behavior here by setting a
+ # key in the state dict or something.
+ clean_hg_repo(logger, hg_repo_path)
+
+ # We now have the 2 heads locally. Our next step is to create a new
+ # commit by doing a merge. This requires a checkout to run the merge
+ # command. We use a temporary checkout for this so we don't need to
+ # worry about contention or contamination of the original repo.
+
+ with temporary_git_checkout(git_repo, base_commit.id, branch='base') \
+ as temp_git_repo:
+
+ state['temp_git_repo'] = temp_git_repo
+
+ # Ensure the base revision is synchronized with Mercurial. This will
+ # produce an extra commit in Mercurial. But it ensures the "diff"
+ # for the commit corresponding to the Git head's "merge" doesn't
+ # contain unwanted changes. This makes it easier to
+ # graft/cherry-pick, should someone want to do that.
+ commit_needed = copy_git_workdir_to_hg_workdir(
+ temp_git_repo, hg_repo_path, dir_map=dir_map)
+
+ yield state, commit_needed
+
+ # Do a merge but don't commit it because we only care about
+ # producing a Mercurial changeset and the Git commit isn't required
+ # for that: we just care the working tree is current.
+ logger.warn('performing git merge against %s' % head_commit.id)
+ subprocess.check_call(
+ [b'git', b'merge', b'--no-commit', b'--strategy=recursive',
+ head_commit.id],
+ cwd=temp_git_repo)
+
+ logger.warn('copying Git changes to Mercurial working directory')
+ commit_needed = copy_git_workdir_to_hg_workdir(
+ temp_git_repo, hg_repo_path, dir_map=dir_map)
+
+ yield state, commit_needed
--- a/vcssync/setup.py
+++ b/vcssync/setup.py
@@ -12,16 +12,17 @@ console_scripts = [
]
# These commands are really only useful for testing. So don't expose them by
# default.
if 'VCSSYNC_ENABLE_TESTING_COMMANDS' in os.environ:
console_scripts.extend([
'test-synchronize-directories=mozvcssync.cli:test_synchronize_directories',
'test-copy-workdir-git-to-hg=mozvcssync.cli:test_copy_workdir_git_to_hg',
+ 'test-overlay-git-ref-via-squash=mozvcssync.cli:test_overlay_git_ref_via_squash',
])
# ansible/roles/vcs-sync/defaults/main.yml must be updated if this package's
# version number is changed.
setup(
name='mozvcssync',
version='0.1',
--- a/vcssync/tests/helpers.sh
+++ b/vcssync/tests/helpers.sh
@@ -7,16 +7,19 @@
# make git commits deterministic and environment agnostic
export GIT_AUTHOR_NAME=test
export GIT_AUTHOR_EMAIL=test@example.com
export GIT_AUTHOR_DATE='Thu Jan 1 00:00:00 1970 +0000'
export GIT_COMMITTER_NAME=test
export GIT_COMMITTER_EMAIL=test@example.com
export GIT_COMMITTER_DATE='Thu Jan 1 00:00:00 1970 +0000'
+# Needed for hglib.
+export HGDATE='0 0'
+
export BETAMAX_LIBRARY_DIR=$TESTDIR/vcssync/tests/cassettes
standardgitrepo() {
here=`pwd`
git init $1
cd $1
echo 0 > foo
git add foo
new file mode 100644
--- /dev/null
+++ b/vcssync/tests/test-overlay-git-ref-via-squash.t
@@ -0,0 +1,237 @@
+ $ . $TESTDIR/vcssync/tests/helpers.sh
+ $ mkdir tmpdir
+
+ $ standardgitrepo grepo > /dev/null 2>&1
+
+ $ git clone grepo grepo-head
+ Cloning into 'grepo-head'...
+ done.
+ $ cd grepo-head
+ $ echo 1 > bar
+ $ git add bar
+ $ git commit -m 'add bar'
+ [master 697a4b8] add bar
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+ $ git log --graph --format=oneline
+ * 697a4b81e1cc0c2bbd5daa07544896a85f17c939 add bar
+ * a447b9b0ff25bf17daab1c7edae4a998eca0adac dummy commit 1 after merge
+ * fc30a4fbd1fe16d4c84ca50119e0c404c13967a3 Merge branch 'head2'
+ |\
+ | * 7b4eab003357aaa2873a1976cb86f5c4a70f5f22 dummy commit 2 on head2
+ | * 8b358b77dede1a070e1047874be3679a31a210c9 dummy commit 1 on head2
+ * | 7a13658c4512ce4c99800417db933f4a1d3fdcb3 dummy commit 2 on master
+ * | 85fd94699e69ce4d2d55171078541c1019f111e4 dummy commit 1 on master
+ |/
+ * ecba8e9490aa2f14345a2da0da62631928ff2968 create file1-20, file1-50 and file1-80 as copies with mods
+ * 2b77427ac0fe55e172d4174530c9bcc4b2544ff6 copy file0-moved and rename source
+ * 9fea386651d90b505d5d1fa2e70c465562b04c7d move file0 to file0-moved
+ * 4d2fb4a1b4defb3cdf7abb52d9d3c91245d26194 copy file0 to file0-copy1 and file0-copy2
+ * 73617395f86af21dde35a52e6149c8e1aac4e68f copy file0 to file0-copy0
+ * 6044be85e82c72f9115362f88c42ce94016e8718 add file0 and file1
+ * dbd62b82aaf0a7a05665d9455a9b4d490d52ddaf initial
+
+ $ cd ..
+
+ $ git init --bare localgrepo
+ Initialized empty Git repository in $TESTTMP/localgrepo/
+ $ hg init hgsource
+ $ cd hgsource
+ $ echo 0 > foo
+ $ hg -q commit -A -m initial
+ $ cd ..
+
+ $ test-overlay-git-ref-via-squash \
+ > file://$TESTTMP/grepo master \
+ > file://$TESTTMP/grepo-head master \
+ > file://$TESTTMP/hgsource default \
+ > localgrepo localhrepo \
+ > --map .:subdir
+ fetching master from file://$TESTTMP/grepo
+ From file://$TESTTMP/grepo
+ * [new branch] master -> refs/overlay/*/base (glob)
+ fetching master from file://$TESTTMP/grepo-head
+ From file://$TESTTMP/grepo-head
+ * [new branch] master -> refs/overlay/*/head (glob)
+ executing: hg pull -r default file://$TESTTMP/hgsource
+ hg> pulling from file://$TESTTMP/hgsource
+ hg> adding changesets
+ hg> adding manifests
+ hg> adding file changes
+ hg> added 1 changesets with 1 changes to 1 files
+ hg> (run 'hg update' to get a working copy)
+ executing: hg update default
+ hg> 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ reverting all local changes and purging localhrepo
+ executing: hg --quiet revert --no-backup --all
+ executing: hg purge --all
+ creating temporary Git checkout in $TESTTMP/tmpdir/git-temp-checkout-* (glob)
+ Cloning into '$TESTTMP/tmpdir/git-temp-checkout-*'... (glob)
+ done.
+ Switched to a new branch 'base'
+ git base commit: a447b9b0ff25bf17daab1c7edae4a998eca0adac
+ git head commit: 697a4b81e1cc0c2bbd5daa07544896a85f17c939
+ committing in Mercurial to synchronize repo with Git base revision
+ committed d260797df3042a696fe09293e8d89f60e55d5721
+ performing git merge against 697a4b81e1cc0c2bbd5daa07544896a85f17c939
+ Updating a447b9b..697a4b8
+ Fast-forward
+ bar | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+ copying Git changes to Mercurial working directory
+ committing in Mercurial to synchronize repo with Git head revision
+ committed db492c28605b47ed26519a9bff82722456e832a0
+ removing temporary Git repo $TESTTMP/tmpdir/git-temp-checkout-* (glob)
+
+Refs should not be present on local Git repo
+
+ $ git -C localgrepo for-each-ref
+
+Mercurial repo should contain a synchronization commit and an integration
+commit
+
+ $ hg -R localhrepo log -G -T '{rev}:{node|short} {desc}\n' -p
+ @ 2:db492c28605b integrate head
+ | diff -r d260797df304 -r db492c28605b subdir/bar
+ | --- a/subdir/bar Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/bar Thu Jan 01 00:00:00 1970 +0000
+ | @@ -1,1 +1,1 @@
+ | -4
+ | +1
+ |
+ o 1:d260797df304 synchronize base
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/bar
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/bar Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,1 @@
+ | +4
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file0-copied-with-move
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file0-copied-with-move Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,11 @@
+ | +file0 0
+ | +file0 1
+ | +file0 2
+ | +file0 3
+ | +file0 4
+ | +file0 5
+ | +file0 6
+ | +file0 7
+ | +file0 8
+ | +file0 9
+ | +file0 10
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file0-copy0
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file0-copy0 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,11 @@
+ | +file0 0
+ | +file0 1
+ | +file0 2
+ | +file0 3
+ | +file0 4
+ | +file0 5
+ | +file0 6
+ | +file0 7
+ | +file0 8
+ | +file0 9
+ | +file0 10
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file0-copy1
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file0-copy1 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,11 @@
+ | +file0 0
+ | +file0 1
+ | +file0 2
+ | +file0 3
+ | +file0 4
+ | +file0 5
+ | +file0 6
+ | +file0 7
+ | +file0 8
+ | +file0 9
+ | +file0 10
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file0-copy2
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file0-copy2 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,11 @@
+ | +file0 0
+ | +file0 1
+ | +file0 2
+ | +file0 3
+ | +file0 4
+ | +file0 5
+ | +file0 6
+ | +file0 7
+ | +file0 8
+ | +file0 9
+ | +file0 10
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file0-moved-with-copy
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file0-moved-with-copy Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,11 @@
+ | +file0 0
+ | +file0 1
+ | +file0 2
+ | +file0 3
+ | +file0 4
+ | +file0 5
+ | +file0 6
+ | +file0 7
+ | +file0 8
+ | +file0 9
+ | +file0 10
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file1
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file1 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,10 @@
+ | +file1 0
+ | +file1 1
+ | +file1 2
+ | +file1 3
+ | +file1 4
+ | +file1 5
+ | +file1 6
+ | +file1 7
+ | +file1 8
+ | +file1 9
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file1-20
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file1-20 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,2 @@
+ | +file1 2
+ | +file1 7
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file1-50
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file1-50 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,5 @@
+ | +file1 0
+ | +file1 1
+ | +file1 2
+ | +file1 3
+ | +file1 4
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/file1-80
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/file1-80 Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,8 @@
+ | +file1 0
+ | +file1 1
+ | +file1 2
+ | +file1 3
+ | +file1 5
+ | +file1 6
+ | +file1 7
+ | +file1 9
+ | diff -r af1e0a150cd4 -r d260797df304 subdir/foo
+ | --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ | +++ b/subdir/foo Thu Jan 01 00:00:00 1970 +0000
+ | @@ -0,0 +1,1 @@
+ | +5
+ |
+ o 0:af1e0a150cd4 initial
+ diff -r 000000000000 -r af1e0a150cd4 foo
+ --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+ +++ b/foo Thu Jan 01 00:00:00 1970 +0000
+ @@ -0,0 +1,1 @@
+ +0
+
+