hgmo: support fetching clonebundles.manifest during pull (bug 1352494); r?glob draft
authorGregory Szorc <gps@mozilla.com>
Fri, 31 Mar 2017 11:36:18 -0700
changeset 10601 fccc9bac6e5cace263f511485e59d7f604286136
parent 10600 a2ffb994cc579262d075733145b6d5560f08f622
child 10602 d4ad538839194ba976bc76237687617e8ab757b3
push id1596
push userbmo:gps@mozilla.com
push dateFri, 31 Mar 2017 21:38:22 +0000
reviewersglob
bugs1352494
hgmo: support fetching clonebundles.manifest during pull (bug 1352494); r?glob Currently, the bundleclone extension fetches the remote's clone bundles manifest during pull operations and saves it locally. Since we'll be removing the bundleclone extension, we need to implement this logic elsewhere. The hgmo extension is where we are putting our server-side customizations for hg.mo. So it seems a logical place to put this functionality. This code is pretty much a copy of what's in the bundleclone extension. An explicit test has been added for good measure. MozReview-Commit-ID: APyjsnaVGa2
hgext/hgmo/__init__.py
hgext/hgmo/tests/helpers.sh
hgext/hgmo/tests/test-clonebundles-pull.t
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -82,16 +82,17 @@ import types
 from mercurial.i18n import _
 from mercurial.node import bin, short
 from mercurial import (
     bookmarks,
     cmdutil,
     commands,
     encoding,
     error,
+    exchange,
     extensions,
     hg,
     revset,
     templatefilters,
     util,
     wireproto,
 )
 from mercurial.hgweb import (
@@ -607,16 +608,37 @@ def mozbuildinfocommand(ui, repo, *paths
 
     # TODO send data to templater.
     # Use stable output and indentation to make testing easier.
     ui.write(json.dumps(d, indent=2, sort_keys=True))
     ui.write('\n')
     return
 
 
+def pull(orig, repo, remote, *args, **kwargs):
+    """Wraps exchange.pull to fetch the remote clonebundles.manifest."""
+    res = orig(repo, remote, *args, **kwargs)
+
+    if not repo.ui.configbool('hgmo', 'pullclonebundlesmanifest', False):
+        return res
+
+    if not remote.capable('clonebundles'):
+        return res
+
+    lock = repo.lock()
+    repo.ui.status(_('pulling clonebundles manifest\n'))
+    try:
+        manifest = remote._call('clonebundles')
+        repo.vfs.write('clonebundles.manifest', manifest)
+    finally:
+        lock.release()
+
+    return res
+
+
 def processbundlesmanifest(orig, repo, proto):
     """Wraps wireproto.clonebundles.
 
     We examine source IP addresses and advertise URLs for the same
     AWS region if the source is in AWS.
     """
     # Delay import because this extension can be run on local
     # developer machines.
@@ -723,16 +745,17 @@ def filelog(orig, web, req, tmpl):
                 entry['pushdate'] = None
 
         return tmpl(*t.args, **t.kwargs)
     else:
         return orig(web, req, tmpl)
 
 
 def extsetup(ui):
+    extensions.wrapfunction(exchange, 'pull', pull)
     extensions.wrapfunction(webutil, 'changesetentry', changesetentry)
     extensions.wrapfunction(webutil, 'changelistentry', changelistentry)
     extensions.wrapfunction(bookmarks, 'updatefromremote', bmupdatefromremote)
     extensions.wrapfunction(webcommands, 'filelog', filelog)
 
     revset.symbols['reviewer'] = revset_reviewer
     revset.safesymbols.add('reviewer')
 
--- a/hgext/hgmo/tests/helpers.sh
+++ b/hgext/hgmo/tests/helpers.sh
@@ -1,13 +1,14 @@
 startserver() {
   hg init server
   cd server
   cat > .hg/hgrc << EOF
 [extensions]
+clonebundles =
 hgmo = $TESTDIR/hgext/hgmo
 
 [web]
 push_ssl = False
 allow_push = *
 EOF
 
   hg serve -d -p $HGPORT --pid-file hg.pid --hgmo -E error.log
new file mode 100644
--- /dev/null
+++ b/hgext/hgmo/tests/test-clonebundles-pull.t
@@ -0,0 +1,112 @@
+  $ . $TESTDIR/hgext/hgmo/tests/helpers.sh
+
+  $ cat >> $HGRCPATH << EOF
+  > [extensions]
+  > hgmo = $TESTDIR/hgext/hgmo
+  > EOF
+
+  $ startserver
+
+  $ cd server
+  $ touch foo
+  $ hg -q commit -A -m initial
+  $ cd ..
+
+  $ hg -q clone http://localhost:$HGPORT repo
+  $ cd repo
+
+clonebundles.manifest should not be transferred by default
+
+  $ hg pull
+  pulling from http://localhost:$HGPORT/
+  searching for changes
+  no changes found
+  $ ls .hg
+  00changelog.i
+  branch
+  cache
+  dirstate
+  hgrc
+  requires
+  store
+  undo.bookmarks
+  undo.branch
+  undo.desc
+  undo.dirstate
+
+Even if enabled and the server doesn't have a clonebundles.manifest
+
+  $ hg --config hgmo.pullclonebundlesmanifest=true pull
+  pulling from http://localhost:$HGPORT/
+  searching for changes
+  no changes found
+  $ ls .hg
+  00changelog.i
+  branch
+  cache
+  dirstate
+  hgrc
+  requires
+  store
+  undo.bookmarks
+  undo.branch
+  undo.desc
+  undo.dirstate
+
+Sanity check that clone bundles manifest is served properly
+
+  $ cat > ../server/.hg/clonebundles.manifest << EOF
+  > https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  > https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  > https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  > https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  > https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+  > EOF
+
+  $ http --no-headers http://localhost:$HGPORT/?cmd=clonebundles
+  200
+  
+  https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+  
+
+clonebundles.manifest should not be transferred by default
+
+  $ hg pull
+  pulling from http://localhost:$HGPORT/
+  searching for changes
+  no changes found
+
+  $ cat .hg/clonebundles.manifest
+  cat: .hg/clonebundles.manifest: No such file or directory
+  [1]
+  $ ls .hg
+  00changelog.i
+  branch
+  cache
+  dirstate
+  hgrc
+  requires
+  store
+  undo.bookmarks
+  undo.branch
+  undo.desc
+  undo.dirstate
+
+enabling config option pulls the manifest
+
+  $ hg --config hgmo.pullclonebundlesmanifest=true pull
+  pulling from http://localhost:$HGPORT/
+  searching for changes
+  no changes found
+  pulling clonebundles manifest
+
+  $ cat .hg/clonebundles.manifest
+  https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+  https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+  https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+  https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+  https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1