--- a/hgext/robustcheckout/__init__.py
+++ b/hgext/robustcheckout/__init__.py
@@ -26,16 +26,17 @@ from mercurial.i18n import _
from mercurial.node import hex, nullid
from mercurial import (
commands,
error,
exchange,
extensions,
cmdutil,
hg,
+ match as matchmod,
registrar,
scmutil,
util,
)
testedwith = '3.7 3.8 3.9 4.0 4.1 4.2 4.3'
minimumhgversion = '3.7'
@@ -53,16 +54,21 @@ else:
def getvfs():
try:
from mercurial.vfs import vfs
return vfs
except ImportError:
return scmutil.vfs
+def getsparse():
+ from mercurial import sparse
+ return sparse
+
+
if os.name == 'nt':
import ctypes
# Get a reference to the DeleteFileW function
# DeleteFileW accepts filenames encoded as a null terminated sequence of
# wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
# strings to null terminated UTF-16 strings.
# However, we receive (byte) strings from mercurial. When these are passed
@@ -134,21 +140,23 @@ def purgewrapper(orig, ui, *args, **kwar
@command('robustcheckout', [
('', 'upstream', '', 'URL of upstream repo to clone from'),
('r', 'revision', '', 'Revision to check out'),
('b', 'branch', '', 'Branch to check out'),
('', 'purge', False, 'Whether to purge the working directory'),
('', 'sharebase', '', 'Directory where shared repos should be placed'),
('', 'networkattempts', 3, 'Maximum number of attempts for network '
'operations'),
+ ('', 'sparseprofile', '', 'Sparse checkout profile to use (path in repo)'),
],
'[OPTION]... URL DEST',
norepo=True)
def robustcheckout(ui, url, dest, upstream=None, revision=None, branch=None,
- purge=False, sharebase=None, networkattempts=None):
+ purge=False, sharebase=None, networkattempts=None,
+ sparseprofile=None):
"""Ensure a working copy has the specified revision checked out.
Repository data is automatically pooled into the common directory
specified by ``--sharebase``, which is a required argument. It is required
because pooling storage prevents excessive cloning, which makes operations
complete faster.
One of ``--revision`` or ``--branch`` must be specified. ``--revision``
@@ -158,16 +166,26 @@ def robustcheckout(ui, url, dest, upstre
If ``--upstream`` is used, the repo at that URL is used to perform the
initial clone instead of cloning from the repo where the desired revision
is located.
``--purge`` controls whether to removed untracked and ignored files from
the working directory. If used, the end state of the working directory
should only contain files explicitly under version control for the requested
revision.
+
+ ``--sparseprofile`` can be used to specify a sparse checkout profile to use.
+ The sparse checkout profile corresponds to a file in the revision to be
+ checked out. If a previous sparse profile or config is present, it will be
+ replaced by this sparse profile. We choose not to "widen" the sparse config
+ so operations are as deterministic as possible. If an existing checkout
+ is present and it isn't using a sparse checkout, we error. This is to
+ prevent accidentally enabling sparse on a repository that may have
+ clients that aren't sparse aware. Sparse checkout support requires Mercurial
+ 4.3 or newer and the ``sparse`` extension must be enabled.
"""
if not revision and not branch:
raise error.Abort('must specify one of --revision or --branch')
if revision and branch:
raise error.Abort('cannot specify both --revision and --branch')
# Require revision to look like a SHA-1.
@@ -176,16 +194,33 @@ def robustcheckout(ui, url, dest, upstre
raise error.Abort('--revision must be a SHA-1 fragment 12-40 '
'characters long')
sharebase = sharebase or ui.config('share', 'pool')
if not sharebase:
raise error.Abort('share base directory not defined; refusing to operate',
hint='define share.pool config option or pass --sharebase')
+ # Sparse profile support was added in Mercurial 4.3, where it was highly
+ # experimental. Because of the fragility of it, we only support sparse
+ # profiles on 4.3. When 4.4 is released, we'll need to opt in to sparse
+ # support. We /could/ silently fall back to non-sparse when not supported.
+ # However, given that sparse has performance implications, we want to fail
+ # fast if we can't satisfy the desired checkout request.
+ if sparseprofile:
+ if util.versiontuple(n=2) != (4, 3):
+ raise error.Abort('sparse profile support only available for '
+ 'Mercurial 4.3 (using %s)' % util.version())
+
+ try:
+ extensions.find('sparse')
+ except KeyError:
+ raise error.Abort('sparse extension must be enabled to use '
+ '--sparseprofile')
+
ui.warn('(using Mercurial %s)\n' % util.version())
# worker.backgroundclose only makes things faster if running anti-virus,
# which our automation doesn't. Disable it.
ui.setconfig('worker', 'backgroundclose', False)
# By default the progress bar starts after 3s and updates every 0.1s. We
# change this so it shows and updates every 1.0s.
@@ -195,26 +230,30 @@ def robustcheckout(ui, url, dest, upstre
# otherwise we're at the whim of whatever configs are used in automation.
ui.setconfig('progress', 'delay', 1.0)
ui.setconfig('progress', 'refresh', 1.0)
ui.setconfig('progress', 'assume-tty', True)
sharebase = os.path.realpath(sharebase)
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
- sharebase, networkattempts)
+ sharebase, networkattempts,
+ sparse_profile=sparseprofile)
+
def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase,
- networkattemptlimit, networkattempts=None):
+ networkattemptlimit, networkattempts=None, sparse_profile=None):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
- sharebase, networkattemptlimit, networkattempts)
+ sharebase, networkattemptlimit,
+ networkattempts=networkattempts,
+ sparse_profile=sparse_profile)
ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
dest))
# We assume that we're the only process on the machine touching the
# repository paths that we were told to use. This means our recovery
# scenario when things aren't "right" is to just nuke things and start
# from scratch. This is easier to implement than verifying the state
@@ -230,16 +269,30 @@ def _docheckout(ui, url, dest, upstream,
storepath = os.path.dirname(storepath)
storevfs = getvfs()(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
+ # Refuse to enable sparse checkouts on existing checkouts. The reasoning
+ # here is that another consumer of this repo may not be sparse aware. If we
+ # enabled sparse, we would lock them out.
+ if destvfs.exists() and sparse_profile and not destvfs.exists('.hg/sparse'):
+ raise error.Abort('cannot enable sparse profile on existing '
+ 'non-sparse checkout',
+ hint='use a separate working directory to use sparse')
+
+ # And the other direction for symmetry.
+ if not sparse_profile and destvfs.exists('.hg/sparse'):
+ raise error.Abort('cannot use non-sparse checkout on existing sparse '
+ 'checkout',
+ hint='use a separate working directory to use sparse')
+
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
@@ -481,24 +534,78 @@ def _docheckout(ui, url, dest, upstream,
# working directory manipulation.
# Purge if requested. We purge before update because this way we're
# guaranteed to not have conflicts on `hg update`.
if purge and not created:
ui.write('(purging working directory)\n')
purgeext = extensions.find('purge')
- if purgeext.purge(ui, repo, all=True, abort_on_err=True,
- # The function expects all arguments to be
- # defined.
- **{'print': None, 'print0': None, 'dirs': None,
- 'files': None}):
- raise error.Abort('error purging')
+ # Mercurial 4.3 doesn't purge files outside the sparse checkout.
+ # See https://bz.mercurial-scm.org/show_bug.cgi?id=5626. Force
+ # purging by monkeypatching the sparse matcher.
+ try:
+ old_sparse_fn = getattr(repo.dirstate, '_sparsematchfn', None)
+ if old_sparse_fn is not None:
+ assert util.versiontuple(n=2) == (4, 3)
+ repo.dirstate._sparsematchfn = lambda: matchmod.always(repo.root, '')
+
+ if purgeext.purge(ui, repo, all=True, abort_on_err=True,
+ # The function expects all arguments to be
+ # defined.
+ **{'print': None, 'print0': None, 'dirs': None,
+ 'files': None}):
+ raise error.Abort('error purging')
+ finally:
+ if old_sparse_fn is not None:
+ repo.dirstate._sparsematchfn = old_sparse_fn
# Update the working directory.
+
+ if sparse_profile:
+ sparsemod = getsparse()
+
+ # By default, Mercurial will ignore unknown sparse profiles. This could
+ # lead to a full checkout. Be more strict.
+ try:
+ repo.filectx(sparse_profile, changeid=checkoutrevision).data()
+ except error.ManifestLookupError:
+ raise error.Abort('sparse profile %s does not exist at revision '
+ '%s' % (sparse_profile, checkoutrevision))
+
+ old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread('sparse'))
+ old_includes, old_excludes, old_profiles = old_config
+
+ if old_profiles == {sparse_profile} and not old_includes and not \
+ old_excludes:
+ ui.write('(sparse profile %s already set; no need to update '
+ 'sparse config)\n' % sparse_profile)
+ else:
+ if old_includes or old_excludes or old_profiles:
+ ui.write('(replacing existing sparse config with profile '
+ '%s)\n' % sparse_profile)
+ else:
+ ui.write('(setting sparse config to profile %s)\n' %
+ sparse_profile)
+
+ # If doing an incremental update, this will perform two updates:
+ # one to change the sparse profile and another to update to the new
+ # revision. This is not desired. But there's not a good API in
+ # Mercurial to do this as one operation.
+ with repo.wlock():
+ fcounts = map(len, sparsemod._updateconfigandrefreshwdir(
+ repo, [], [], [sparse_profile], force=True))
+
+ fcounts = tuple(list(fcounts) + [0])
+ repo.ui.status(_('%d files updated, %d files merged, '
+ '%d files removed, %d files unresolved\n') %
+ fcounts)
+
+ ui.write('(sparse refresh complete)\n')
+
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort('error updating')
ui.write('updated to %s\n' % checkoutrevision)
return None
def extsetup(ui):
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-sparse.t
@@ -0,0 +1,218 @@
+ $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Add some files to server repo to test sparseness
+
+ $ cd server/repo0
+ $ hg -q up default
+ $ mkdir dir0 dir1
+ $ touch dir0/foo.c dir0/foo.h dir0/foo.py dir1/foo.py dir1/bar.py
+ $ hg -q commit -A -m 'add some files to test sparse'
+ $ mkdir profiles
+ $ cat > profiles/python << EOF
+ > [include]
+ > glob:**/*.py
+ > EOF
+ $ cat > profiles/dir0 << EOF
+ > [include]
+ > glob:dir0/**
+ > EOF
+ $ cat > profiles/dir1 << EOF
+ > [include]
+ > glob:dir1/**
+ > EOF
+ $ hg -q commit -A -m 'add sparse profiles'
+ $ touch dir1/baz.py
+ $ hg -q commit -A -m 'add dir1/baz.py'
+ $ cd ../..
+
+#if no-hg43+
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b --sparseprofile foo
+ abort: sparse profile support only available for Mercurial 4.3 (using *) (glob)
+ [255]
+
+#endif
+
+#if hg43+
+
+Attempting a sparse checkout without sparse extension results in error
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 no-ext --revision 6af47298a235 --sparseprofile irrelevant
+ abort: sparse extension must be enabled to use --sparseprofile
+ [255]
+
+ $ cat >> $HGRCPATH << EOF
+ > [extensions]
+ > sparse=
+ > EOF
+
+Enabling a sparse profile on a repo not using sparse is an error
+
+ $ hg -q robustcheckout http://localhost:$HGPORT/repo0 no-sparse --revision 6af47298a235
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at no-sparse
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 no-sparse --revision 6af47298a235 --sparseprofile irrelevant
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at no-sparse
+ abort: cannot enable sparse profile on existing non-sparse checkout
+ (use a separate working directory to use sparse)
+ [255]
+
+Specifying a sparse profile that doesn't exist results in error
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 bad-profile --revision 6af47298a235 --sparseprofile doesnotexist
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at bad-profile
+ (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+ searching for changes
+ no changes found
+ abort: sparse profile doesnotexist does not exist at revision 6af47298a235491213679cd4f91881d22c735c72
+ [255]
+
+Specifying a sparse profile uses it
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-simple --revision 6af47298a235 --sparseprofile profiles/python
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-simple
+ (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+ searching for changes
+ no changes found
+ (setting sparse config to profile profiles/python)
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ (sparse refresh complete)
+ warning: sparse profile 'profiles/python' not found in rev 000000000000 - ignoring it
+ 3 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+ $ hg -R sparse-simple files
+ sparse-simple/dir0/foo.py
+ sparse-simple/dir1/bar.py
+ sparse-simple/dir1/foo.py
+
+No-op update does something reasonable
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-simple --revision 6af47298a235 --sparseprofile profiles/python
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-simple
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (sparse profile profiles/python already set; no need to update sparse config)
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+Attempting to remove sparse from a sparse checkout is not allowed
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-simple --revision 4916c5373fd6
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@4916c5373fd6 is available at sparse-simple
+ abort: cannot use non-sparse checkout on existing sparse checkout
+ (use a separate working directory to use sparse)
+ [255]
+
+Specifying a new sparse profile will replace existing profile
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-simple --revision 6af47298a235 --sparseprofile profiles/dir0
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-simple
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (replacing existing sparse config with profile profiles/dir0)
+ 2 files updated, 2 files merged, 0 files removed, 0 files unresolved
+ (sparse refresh complete)
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+ $ hg -R sparse-simple files
+ sparse-simple/dir0/foo.c
+ sparse-simple/dir0/foo.h
+ sparse-simple/dir0/foo.py
+
+Specifying a new sparse profile and updating the revision works
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-simple --revision 4916c5373fd6 --sparseprofile profiles/dir1
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@4916c5373fd6 is available at sparse-simple
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (replacing existing sparse config with profile profiles/dir1)
+ 2 files updated, 3 files merged, 0 files removed, 0 files unresolved
+ (sparse refresh complete)
+ 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 4916c5373fd67aca0412e74b0ffeeb0291a86bfd
+
+ $ hg -R sparse-simple files
+ sparse-simple/dir1/bar.py
+ sparse-simple/dir1/baz.py
+ sparse-simple/dir1/foo.py
+
+Purging a file outside the sparse profile works
+
+ $ hg -q robustcheckout http://localhost:$HGPORT/repo0 sparse-purge --revision 6af47298a235 --sparseprofile profiles/dir0
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-purge
+ (setting sparse config to profile profiles/dir0)
+ (sparse refresh complete)
+ warning: sparse profile 'profiles/dir0' not found in rev 000000000000 - ignoring it
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+Purging with update to same revision
+
+ $ mkdir sparse-purge/dir1 sparse-purge/dir2
+ $ touch sparse-purge/dir0/extrafile sparse-purge/dir1/extrafile sparse-purge/dir2/extrafile
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-purge --revision 6af47298a235 --sparseprofile profiles/dir0 --purge
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-purge
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (purging working directory)
+ (sparse profile profiles/dir0 already set; no need to update sparse config)
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+ $ find sparse-purge -type f -name extrafile
+ $ hg --cwd sparse-purge status --all
+ C dir0/foo.c
+ C dir0/foo.h
+ C dir0/foo.py
+
+Purging with update to different revision
+
+ $ mkdir sparse-purge/dir1 sparse-purge/dir2
+ $ touch sparse-purge/dir0/extrafile sparse-purge/dir1/extrafile sparse-purge/dir2/extrafile
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-purge --revision 4916c5373fd6 --sparseprofile profiles/dir0 --purge
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@4916c5373fd6 is available at sparse-purge
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (purging working directory)
+ (sparse profile profiles/dir0 already set; no need to update sparse config)
+ 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+ updated to 4916c5373fd67aca0412e74b0ffeeb0291a86bfd
+
+ $ find sparse-purge -type f -name extrafile
+ $ hg --cwd sparse-purge status --all
+ C dir0/foo.c
+ C dir0/foo.h
+ C dir0/foo.py
+
+Purge with update to different revision and profile
+
+ $ mkdir sparse-purge/dir1 sparse-purge/dir2
+ $ touch sparse-purge/dir0/extrafile sparse-purge/dir1/extrafile sparse-purge/dir2/extrafile
+
+ $ hg robustcheckout http://localhost:$HGPORT/repo0 sparse-purge --revision 6af47298a235 --sparseprofile profiles/dir1 --purge
+ (using Mercurial *) (glob)
+ ensuring http://localhost:$HGPORT/repo0@6af47298a235 is available at sparse-purge
+ (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+ (purging working directory)
+ (replacing existing sparse config with profile profiles/dir1)
+ 3 files updated, 3 files merged, 0 files removed, 0 files unresolved
+ (sparse refresh complete)
+ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+ updated to 6af47298a235491213679cd4f91881d22c735c72
+
+ $ find sparse-purge -type f -name extrafile
+ $ hg --cwd sparse-purge status --all
+ C dir1/bar.py
+ C dir1/foo.py
+
+#endif