manualoverlay: add extension for manually creating overlay commits (Bug 1374534). r?gps draft
authorSteven MacLeod <smacleod@mozilla.com>
Thu, 22 Jun 2017 15:19:29 -0400
changeset 11336 ee7844dbb378b3f706bc39d49a0c2c986981e0cf
parent 11335 647c9984a33e011eeb522721812f915ba8804545
push id1720
push usersmacleod@mozilla.com
push dateTue, 27 Jun 2017 22:26:13 +0000
reviewersgps
bugs1374534
manualoverlay: add extension for manually creating overlay commits (Bug 1374534). r?gps This extension can be used to resync when the overlay repo becomes out of sync due to the race condition between backout pushes and homu landing other changes. After this rogue commit has been landed and linearized, it will fail to overlay due to file difference in the overlayed directory. Using the pull request id of the rogue change, the corresponding linearized commit is found and used for the commit's message and extra data fields. The user must provide file modifications that will resync the overlayed sub directory manually. The created commit can then be pushed up to the overlay target and overlay will assume it was the problematic changeset but containing files that will match going forward. This change has been tested manually for the most part as the current code relies on network access to hg.mozilla.org. A future change can add automated tests for this without much trouble using something like betamax to record this interaction. MozReview-Commit-ID: E501R5j2c7Y
hgext/manualoverlay/__init__.py
hgext/manualoverlay/tests/helpers.sh
hgext/manualoverlay/tests/test-pr-url-validation.t
testing/vcttesting/testing.py
new file mode 100644
--- /dev/null
+++ b/hgext/manualoverlay/__init__.py
@@ -0,0 +1,130 @@
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import os
+import re
+import urllib
+
+from mercurial.i18n import _
+from mercurial import (
+    cmdutil,
+    commands,
+    demandimport,
+    encoding,
+    error,
+    extensions,
+)
+
+OUR_DIR = os.path.normpath(os.path.dirname(__file__))
+execfile(os.path.join(OUR_DIR, '..', 'bootstrap.py'))
+
+# requests doesn't like lazy importing
+with demandimport.deactivated():
+    import requests  # noqa
+
+
+testedwith = '4.1 4.2'
+
+cmdtable = {}
+command = cmdutil.command(cmdtable)
+
+REVISION_KEY = 'subtree_revision'
+SOURCE_KEY = 'subtree_source'
+LINEAR_REPO_URL = 'https://hg.mozilla.org/projects/converted-servo-linear'
+GITHUB_PR_URL = re.compile(
+    r'https://github\.com/servo/servo/pull/(?P<pr>[1-9][0-9]*)/?')
+COMMIT_MSG_PR = re.compile(r'servo: Merge #(?P<pr>[1-9][0-9]*) - .*')
+
+
+def commitcommand(orig, ui, repo, *args, **kwargs):
+    pr_url = kwargs.get('manualservosync')
+    if not pr_url:
+        return orig(ui, repo, *args, **kwargs)
+
+    m = GITHUB_PR_URL.match(pr_url)
+    if m is None:
+        raise error.Abort(
+            _('--manualservosync was not a proper github pull request url'),
+            hint=_('url must be to a servo/servo pull request of the form '
+                   'https://github.com/servo/servo/pull/<pr-number>'))
+
+    pr = m.group('pr')
+
+    revset = urllib.quote('keyword("servo: Merge #%s")' % pr)
+    url = '%s/json-log?rev=%s' % (LINEAR_REPO_URL, revset)
+    r = requests.get(url)
+    commits = r.json()['entries']
+
+    if not commits:
+        raise error.Abort(
+            _('could not find linearized commit corresponding to %s' % pr_url),
+            hint=_('If this pull requests has recently been merged it '
+                   'may not be linearized yet, please try again soon'))
+
+    repo.manualsync_commit = {
+        'desc': encoding.tolocal(commits[0]['desc'].encode('utf-8')),
+        'node': encoding.tolocal(commits[0]['node'].encode('utf-8')),
+        'user': encoding.tolocal(commits[0]['user'].encode('utf-8')),
+    }
+    repo.manualsync_pr = pr
+    repo.manualsync = True
+    return orig(ui, repo, *args, **kwargs)
+
+
+def reposetup(ui, repo):
+    if not repo.local():
+        return
+
+    class servosyncrepo(repo.__class__):
+        def commit(self, *args, **kwargs):
+            if not self.manualsync:
+                return super(servosyncrepo, self).commit(*args, **kwargs)
+
+            # Override some of the commit meta data.
+            msg = self.manualsync_commit['desc']
+            user = self.manualsync_commit['user']
+
+            # This method has many keyword arguments that mercurial
+            # ocassionally passes positionally, meanig they end up
+            # in *args, instead of **kwargs. This can be problematic as
+            # naively modifying the value in **kwargs will result in
+            # the argument being passed twice, which is an error.
+            # Protect against this by stripping the values out of
+            # *args and **kwargs, passing them positionally ourselves.
+            for key in ('text', 'user'):
+                if args:
+                    args = args[1:]
+
+                if key in kwargs:
+                    del kwargs[key]
+
+            kwargs['extra'] = kwargs['extra'] if 'extra' in kwargs else {}
+            kwargs['extra'][SOURCE_KEY] = encoding.tolocal(LINEAR_REPO_URL)
+            kwargs['extra'][REVISION_KEY] = self.manualsync_commit['node']
+
+            # TODO: Verify that the file changes being committed are only
+            # under the servo/ directory.
+            ret = super(servosyncrepo, self).commit(
+                msg, user, *args, **kwargs)
+
+            ctx = repo[ret]
+
+            if any(not f.startswith('servo/') for f in ctx.files()):
+                self.ui.warn(
+                    _('warning: this commit touches files outside the servo '
+                      'directory and would be rejected by the server\n'))
+
+            return ctx
+
+    repo.__class__ = servosyncrepo
+    repo.manualsync = False
+
+
+def extsetup(ui):
+    entry = extensions.wrapcommand(commands.table, 'commit', commitcommand)
+    options = entry[1]
+    options.append(
+        ('', 'manualservosync', '',
+         'manually overlay and sync a servo pull request', _("GH_PR_URL")))
new file mode 100644
--- /dev/null
+++ b/hgext/manualoverlay/tests/helpers.sh
@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+export HGUSER='Test User <someone@example.com>'
+
+cat >> $HGRCPATH << EOF
+[extensions]
+strip =
+manualoverlay = $TESTDIR/hgext/manualoverlay
+
+[diff]
+git = true
+EOF
new file mode 100644
--- /dev/null
+++ b/hgext/manualoverlay/tests/test-pr-url-validation.t
@@ -0,0 +1,40 @@
+  $ . $TESTDIR/hgext/manualoverlay/tests/helpers.sh
+
+  $ hg init source
+  $ cd source
+  $ echo "file1" > file1
+  $ hg add file1
+  $ hg commit -m "Initial commit"
+  $ echo "updated-file1" > file1
+
+Test passing a non-url
+  $ hg commit --manualservosync "not-a-well-formed-url"
+  abort: --manualservosync was not a proper github pull request url
+  (url must be to a servo/servo pull request of the form https://github.com/servo/servo/pull/<pr-number>)
+  [255]
+
+Test passing a url to a non servo/servo pull request
+  $ hg commit --manualservosync "https://github.com/wrong/repo/pull/17455"
+  abort: --manualservosync was not a proper github pull request url
+  (url must be to a servo/servo pull request of the form https://github.com/servo/servo/pull/<pr-number>)
+  [255]
+
+Test passing a url with an alphanumeric pull request id
+  $ hg commit --manualservosync "https://github.com/servo/servo/pull/abc123"
+  abort: --manualservosync was not a proper github pull request url
+  (url must be to a servo/servo pull request of the form https://github.com/servo/servo/pull/<pr-number>)
+  [255]
+
+Committing without --manualservosync should still work
+  $ hg commit -m "Normal Commit"
+  $ hg log -r .
+  changeset:   1:94650105917f
+  tag:         tip
+  user:        Test User <someone@example.com>
+  date:        Thu Jan 01 00:00:00 1970 +0000
+  summary:     Normal Commit
+  
+
+Overlay extra data shouldn't exist on a normal commit
+  $ hg log -r . --template '{extras}'
+  branch=default (no-eol)
--- a/testing/vcttesting/testing.py
+++ b/testing/vcttesting/testing.py
@@ -34,16 +34,19 @@ PYTHON_COVERAGE_DIRS = (
 # by the Python test harness.
 UNIT_TEST_DIRS = {
     'autoland/tests': {
         'venvs': {'global'},
     },
     'git/tests': {
         'venvs': {'global'},
     },
+    'hgext/manualoverlay': {
+        'venvs': {'vcssync'},
+    },
     'hgext/overlay': {
         'venvs': {'vcssync'},
     },
     'hgserver/tests': {
         'venvs': {'global'},
     },
     'pylib': {
         'venvs': {'global', 'hgdev'},