hgmo: support fetching clonebundles.manifest during pull (
bug 1352494); r?glob
Currently, the bundleclone extension fetches the remote's clone bundles
manifest during pull operations and saves it locally. Since we'll be
removing the bundleclone extension, we need to implement this logic
elsewhere.
The hgmo extension is where we are putting our server-side
customizations for hg.mo. So it seems a logical place to put this
functionality.
This code is pretty much a copy of what's in the bundleclone
extension.
An explicit test has been added for good measure.
MozReview-Commit-ID: APyjsnaVGa2
--- a/hgext/hgmo/__init__.py
+++ b/hgext/hgmo/__init__.py
@@ -82,16 +82,17 @@ import types
from mercurial.i18n import _
from mercurial.node import bin, short
from mercurial import (
bookmarks,
cmdutil,
commands,
encoding,
error,
+ exchange,
extensions,
hg,
revset,
templatefilters,
util,
wireproto,
)
from mercurial.hgweb import (
@@ -607,16 +608,37 @@ def mozbuildinfocommand(ui, repo, *paths
# TODO send data to templater.
# Use stable output and indentation to make testing easier.
ui.write(json.dumps(d, indent=2, sort_keys=True))
ui.write('\n')
return
+def pull(orig, repo, remote, *args, **kwargs):
+ """Wraps exchange.pull to fetch the remote clonebundles.manifest."""
+ res = orig(repo, remote, *args, **kwargs)
+
+ if not repo.ui.configbool('hgmo', 'pullclonebundlesmanifest', False):
+ return res
+
+ if not remote.capable('clonebundles'):
+ return res
+
+ lock = repo.lock()
+ repo.ui.status(_('pulling clonebundles manifest\n'))
+ try:
+ manifest = remote._call('clonebundles')
+ repo.vfs.write('clonebundles.manifest', manifest)
+ finally:
+ lock.release()
+
+ return res
+
+
def processbundlesmanifest(orig, repo, proto):
"""Wraps wireproto.clonebundles.
We examine source IP addresses and advertise URLs for the same
AWS region if the source is in AWS.
"""
# Delay import because this extension can be run on local
# developer machines.
@@ -723,16 +745,17 @@ def filelog(orig, web, req, tmpl):
entry['pushdate'] = None
return tmpl(*t.args, **t.kwargs)
else:
return orig(web, req, tmpl)
def extsetup(ui):
+ extensions.wrapfunction(exchange, 'pull', pull)
extensions.wrapfunction(webutil, 'changesetentry', changesetentry)
extensions.wrapfunction(webutil, 'changelistentry', changelistentry)
extensions.wrapfunction(bookmarks, 'updatefromremote', bmupdatefromremote)
extensions.wrapfunction(webcommands, 'filelog', filelog)
revset.symbols['reviewer'] = revset_reviewer
revset.safesymbols.add('reviewer')
--- a/hgext/hgmo/tests/helpers.sh
+++ b/hgext/hgmo/tests/helpers.sh
@@ -1,13 +1,14 @@
startserver() {
hg init server
cd server
cat > .hg/hgrc << EOF
[extensions]
+clonebundles =
hgmo = $TESTDIR/hgext/hgmo
[web]
push_ssl = False
allow_push = *
EOF
hg serve -d -p $HGPORT --pid-file hg.pid --hgmo -E error.log
new file mode 100644
--- /dev/null
+++ b/hgext/hgmo/tests/test-clonebundles-pull.t
@@ -0,0 +1,112 @@
+ $ . $TESTDIR/hgext/hgmo/tests/helpers.sh
+
+ $ cat >> $HGRCPATH << EOF
+ > [extensions]
+ > hgmo = $TESTDIR/hgext/hgmo
+ > EOF
+
+ $ startserver
+
+ $ cd server
+ $ touch foo
+ $ hg -q commit -A -m initial
+ $ cd ..
+
+ $ hg -q clone http://localhost:$HGPORT repo
+ $ cd repo
+
+clonebundles.manifest should not be transferred by default
+
+ $ hg pull
+ pulling from http://localhost:$HGPORT/
+ searching for changes
+ no changes found
+ $ ls .hg
+ 00changelog.i
+ branch
+ cache
+ dirstate
+ hgrc
+ requires
+ store
+ undo.bookmarks
+ undo.branch
+ undo.desc
+ undo.dirstate
+
+Even if enabled and the server doesn't have a clonebundles.manifest
+
+ $ hg --config hgmo.pullclonebundlesmanifest=true pull
+ pulling from http://localhost:$HGPORT/
+ searching for changes
+ no changes found
+ $ ls .hg
+ 00changelog.i
+ branch
+ cache
+ dirstate
+ hgrc
+ requires
+ store
+ undo.bookmarks
+ undo.branch
+ undo.desc
+ undo.dirstate
+
+Sanity check that clone bundles manifest is served properly
+
+ $ cat > ../server/.hg/clonebundles.manifest << EOF
+ > https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+ > https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+ > https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+ > https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+ > https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+ > EOF
+
+ $ http --no-headers http://localhost:$HGPORT/?cmd=clonebundles
+ 200
+
+ https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+ https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+ https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+ https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+ https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1
+
+
+clonebundles.manifest should not be transferred by default
+
+ $ hg pull
+ pulling from http://localhost:$HGPORT/
+ searching for changes
+ no changes found
+
+ $ cat .hg/clonebundles.manifest
+ cat: .hg/clonebundles.manifest: No such file or directory
+ [1]
+ $ ls .hg
+ 00changelog.i
+ branch
+ cache
+ dirstate
+ hgrc
+ requires
+ store
+ undo.bookmarks
+ undo.branch
+ undo.desc
+ undo.dirstate
+
+enabling config option pulls the manifest
+
+ $ hg --config hgmo.pullclonebundlesmanifest=true pull
+ pulling from http://localhost:$HGPORT/
+ searching for changes
+ no changes found
+ pulling clonebundles manifest
+
+ $ cat .hg/clonebundles.manifest
+ https://hg.cdn.mozilla.net/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 REQUIRESNI=true cdn=true
+ https://s3-us-west-2.amazonaws.com/moz-hg-bundles-us-west-2/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-2
+ https://s3-us-west-1.amazonaws.com/moz-hg-bundles-us-west-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-west-1
+ https://s3-external-1.amazonaws.com/moz-hg-bundles-us-east-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=us-east-1
+ https://s3-eu-central-1.amazonaws.com/moz-hg-bundles-eu-central-1/mozilla-central/77538e1ce4bec5f7aac58a7ceca2da0e38e90a72.zstd.hg BUNDLESPEC=zstd-v2 ec2region=eu-central-1