robustcheckout: add extension to perform clone+checkout robustly (bug 1273305); r?smacleod draft
authorGregory Szorc <gps@mozilla.com>
Mon, 16 May 2016 13:48:26 -0700
changeset 8127 a1591fcfb27c3e409afdfd99543f82a99edc3934
parent 8113 15fd68a53136b3cf8232ba304ad698d2f2f16fb7
push id833
push userbmo:gps@mozilla.com
push dateTue, 17 May 2016 22:54:54 +0000
reviewerssmacleod
bugs1273305, 1270317
robustcheckout: add extension to perform clone+checkout robustly (bug 1273305); r?smacleod Firefox automation has multiple implementations of code that ensures a Mercurial repository is checked out at a specified path. None of these implementations are optimal and there are numerous efficiency gains to be realized. This commit introduces the "robustcheckout" extension. It provides a command (`hg robustcheckout`) which effectively performs a clone/pull + purge + update robustly using optimal techniques for automation. These include the required use of pooled shared stores. The goal is to replace the custom code in automation with this extension, starting with hgtool. The code in this commit comes from the mozharness work I did in bug 1270317. It also incorporates the content of the "purgelong" extension, which was previously a standalone extension only used as part of doing a clone/pull+checkout in automation. MozReview-Commit-ID: 9szgfyqtnHM
hgext/robustcheckout/__init__.py
hgext/robustcheckout/tests/helpers.sh
hgext/robustcheckout/tests/test-abandonded-transaction.t
hgext/robustcheckout/tests/test-corrupt-repo.t
hgext/robustcheckout/tests/test-dest-no-hg.t
hgext/robustcheckout/tests/test-dest-shared-state.t
hgext/robustcheckout/tests/test-missing-parent-dirs.t
hgext/robustcheckout/tests/test-purge.t
hgext/robustcheckout/tests/test-revision-branch.t
hgext/robustcheckout/tests/test-unrelated.t
hgext/robustcheckout/tests/test-update.t
hgext/robustcheckout/tests/test-upstream.t
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/__init__.py
@@ -0,0 +1,311 @@
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+"""Robustly perform a checkout.
+
+This extension provides the ``hg robustcheckout`` command for
+ensuring a working directory is updated to the specified revision
+from a source repo using best practices to ensure optimal clone
+times and storage efficiency.
+"""
+
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import functools
+import os
+import re
+
+from mercurial.i18n import _
+from mercurial import (
+    commands,
+    error,
+    exchange,
+    extensions,
+    cmdutil,
+    hg,
+    scmutil,
+    util,
+)
+
+testedwith = '3.5 3.6 3.7 3.8'
+minimumhgversion = '3.7'
+
+cmdtable = {}
+command = cmdutil.command(cmdtable)
+
+
+if os.name == 'nt':
+    import ctypes
+
+    # Get a reference to the DeleteFileW function
+    # DeleteFileW accepts filenames encoded as a null terminated sequence of
+    # wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
+    # strings to null terminated UTF-16 strings.
+    # However, we receive (byte) strings from mercurial. When these are passed
+    # to DeleteFileW via the c_wchar_p type, they are implicitly decoded via
+    # the 'mbcs' encoding on windows.
+    kernel32 = ctypes.windll.kernel32
+    DeleteFile = kernel32.DeleteFileW
+    DeleteFile.argtypes = [ctypes.c_wchar_p]
+    DeleteFile.restype = ctypes.c_bool
+
+    def unlinklong(fn):
+        normalized_path = '\\\\?\\' + os.path.normpath(fn)
+        if not DeleteFile(normalized_path):
+            raise OSError(errno.EPERM, "couldn't remove long path", fn)
+
+# Not needed on other platforms, but is handy for testing
+else:
+    def unlinklong(fn):
+        os.unlink(fn)
+
+
+def unlinkwrapper(unlinkorig, fn, ui):
+    '''Calls unlink_long if original unlink function fails.'''
+    try:
+        ui.debug('calling unlink_orig %s\n' % fn)
+        return unlinkorig(fn)
+    except WindowsError as e:
+        # Windows error 3 corresponds to ERROR_PATH_NOT_FOUND
+        # only handle this case; re-raise the exception for other kinds of
+        # failures.
+        if e.winerror != 3:
+            raise
+        ui.debug('caught WindowsError ERROR_PATH_NOT_FOUND; '
+                 'calling unlink_long %s\n' % fn)
+        return unlinklong(fn)
+
+
+@contextlib.contextmanager
+def wrapunlink(ui):
+    '''Context manager that temporarily monkeypatches unlink functions.'''
+    purgemod = extensions.find('purge')
+    to_wrap = [(purgemod.util, 'unlink')]
+
+    # Pass along the ui object to the unlink_wrapper so we can get logging out
+    # of it.
+    wrapped = functools.partial(unlinkwrapper, ui=ui)
+
+    # Wrap the original function(s) with our unlink wrapper.
+    originals = {}
+    for mod, func in to_wrap:
+        ui.debug('wrapping %s %s\n' % (mod, func))
+        originals[mod, func] = extensions.wrapfunction(mod, func, wrapped)
+
+    try:
+        yield
+    finally:
+        # Restore the originals.
+        for mod, func in to_wrap:
+            ui.debug('restoring %s %s\n' % (mod, func))
+            setattr(mod, func, originals[mod, func])
+
+
+def purgewrapper(orig, ui, *args, **kwargs):
+    '''Runs original purge() command with unlink monkeypatched.'''
+    with wrapunlink(ui):
+        return orig(ui, *args, **kwargs)
+
+
+@command('robustcheckout', [
+    ('', 'upstream', '', 'URL of upstream repo to clone from'),
+    ('r', 'revision', '', 'Revision to check out'),
+    ('b', 'branch', '', 'Branch to check out'),
+    ('', 'purge', False, 'Whether to purge the working directory'),
+    ('', 'sharebase', '', 'Directory where shared repos should be placed'),
+    ],
+    '[OPTION]... URL DEST',
+    norepo=True)
+def robustcheckout(ui, url, dest, upstream=None, revision=None, branch=None,
+                   purge=False, sharebase=None):
+    """Ensure a working copy has the specified revision checked out."""
+    if not revision and not branch:
+        raise error.Abort('must specify one of --revision or --branch')
+
+    if revision and branch:
+        raise error.Abort('cannot specify both --revision and --branch')
+
+    sharebase = sharebase or ui.config('share', 'pool')
+    if not sharebase:
+        raise error.Abort('share base directory not defined; refusing to operate',
+                          hint='define share.pool config option or pass --sharebase')
+
+    # worker.backgroundclose only makes things faster if running anti-virus,
+    # which our automation doesn't. Disable it.
+    ui.setconfig('worker', 'backgroundclose', False)
+
+    sharebase = os.path.realpath(sharebase)
+
+    return _docheckout(ui, url, dest, upstream, revision, branch, purge,
+                       sharebase)
+
+def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase):
+    def callself():
+        return _docheckout(ui, url, dest, upstream, revision, branch, purge,
+                           sharebase)
+
+    ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
+                                                      dest))
+
+    destvfs = scmutil.vfs(dest, audit=False, realpath=True)
+
+    if destvfs.exists() and not destvfs.exists('.hg'):
+        raise error.Abort('destination exists but no .hg directory')
+
+    # Require checkouts to be tied to shared storage because efficiency.
+    if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
+        ui.warn('(destination is not shared; deleting)\n')
+        destvfs.rmtree(forcibly=True)
+
+    # Verify the shared path exists and is using modern pooled storage.
+    if destvfs.exists('.hg/sharedpath'):
+        storepath = destvfs.read('.hg/sharedpath').strip()
+
+        ui.write('(existing repository shared store: %s)\n' % storepath)
+
+        if not os.path.exists(storepath):
+            ui.warn('(shared store does not exist; deleting)\n')
+            destvfs.rmtree(forcibly=True)
+        elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
+            ui.warn('(shared store does not belong to pooled storage; '
+                    'deleting to improve efficiency)\n')
+            destvfs.rmtree(forcibly=True)
+
+        # FUTURE when we require generaldelta, this is where we can check
+        # for that.
+
+    def deletesharedstore():
+        storepath = destvfs.read('.hg/sharedpath').strip()
+        if storepath.endswith('.hg'):
+            storepath = os.path.dirname(storepath)
+
+        storevfs = scmutil.vfs(storepath, audit=False)
+        storevfs.rmtree(forcibly=True)
+
+    def handlerepoerror(e):
+        if e.message == _('abandoned transaction found'):
+            ui.warn('(abandoned transaction found; trying to recover)\n')
+            repo = hg.repository(ui, dest)
+            if not repo.recover():
+                ui.warn('(could not recover repo state; '
+                        'deleting shared store)\n')
+                deletesharedstore()
+
+            ui.warn('(attempting checkout from beginning)\n')
+            return callself()
+
+        raise
+
+    # At this point we either have an existing working directory using
+    # shared, pooled storage or we have nothing.
+    created = False
+
+    if not destvfs.exists():
+        # Ensure parent directories of destination exist.
+        # Mercurial 3.8 removed ensuredirs and made makedirs race safe.
+        if util.safehasattr(util, 'ensuredirs'):
+            makedirs = util.ensuredirs
+        else:
+            makedirs = util.makedirs
+
+        makedirs(os.path.dirname(destvfs.base), notindexed=True)
+        makedirs(sharebase, notindexed=True)
+
+        if upstream:
+            ui.write('(cloning from upstream repo %s)\n' % upstream)
+        cloneurl = upstream or url
+
+        try:
+            res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
+                           shareopts={'pool': sharebase, 'mode': 'identity'})
+        except error.RepoError as e:
+            return handlerepoerror(e)
+        except error.RevlogError as e:
+            ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
+            deletesharedstore()
+            return callself()
+
+        # TODO retry here.
+        if res is None:
+            raise error.Abort('clone failed')
+
+        # Verify it is using shared pool storage.
+        if not destvfs.exists('.hg/sharedpath'):
+            raise error.Abort('clone did not create a shared repo')
+
+        created = True
+
+    # The destination .hg directory should exist. Now make sure we have the
+    # wanted revision.
+
+    repo = hg.repository(ui, dest)
+    havewantedrev = False
+    if revision:
+        havewantedrev = revision in repo
+    else:
+        assert branch
+        # Branch names are not constant over time, so always pull to
+        # ensure we have the latest revision.
+
+    if not havewantedrev:
+        ui.write('(pulling to obtain %s)\n' % (revision or branch,))
+
+        try:
+            remote = hg.peer(repo, {}, url)
+            pullrevs = [remote.lookup(revision or branch)]
+            pullop = exchange.pull(repo, remote, heads=pullrevs)
+            if not pullop.rheads:
+                raise error.Abort('unable to pull requested revision')
+        except error.Abort as e:
+            if e.message == _('repository is unrelated'):
+                ui.warn('(repository is unrelated; deleting)\n')
+                destvfs.rmtree(forcibly=True)
+                return callself()
+
+            raise
+        except error.RepoError as e:
+            return handlerepoerror(e)
+        except error.RevlogError as e:
+            ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
+            deletesharedstore()
+            return callself()
+        finally:
+            remote.close()
+
+    # Now we should have the wanted revision in the store. Perform
+    # working directory manipulation.
+
+    # Purge if requested. We purge before update because this way we're
+    # guaranteed to not have conflicts on `hg update`.
+    if purge and not created:
+        ui.write('(purging working directory)\n')
+        purgeext = extensions.find('purge')
+
+        if purgeext.purge(ui, repo, all=True, abort_on_err=True,
+                          # The function expects all arguments to be
+                          # defined.
+                          **{'print': None, 'print0': None, 'dirs': None,
+                             'files': None}):
+            raise error.Abort('error purging')
+
+    # Update the working directory.
+    if commands.update(ui, repo, rev=revision or branch, clean=True):
+        raise error.Abort('error updating')
+
+    ctx = repo[revision or branch]
+    ui.write('updated to %s\n' % ctx.hex())
+    return None
+
+
+def extsetup(ui):
+    # Ensure required extensions are loaded.
+    for ext in ('purge', 'share'):
+        try:
+            extensions.find(ext)
+        except KeyError:
+            extensions.load(ui, ext, None)
+
+    purgemod = extensions.find('purge')
+    extensions.wrapcommand(purgemod.cmdtable, 'purge', purgewrapper)
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/helpers.sh
@@ -0,0 +1,41 @@
+cat >> $HGRCPATH << EOF
+[share]
+pool = $TESTTMP/share
+
+[extensions]
+robustcheckout = $TESTDIR/hgext/robustcheckout
+EOF
+
+mkdir server
+hg init server/repo0
+hg init server/repo1
+
+cd server/repo0
+
+touch foo
+hg -q commit -A -m initial0
+echo 1 > foo
+hg commit -m 1
+hg -q up -r 0
+hg -q branch branch1
+echo branch1 > foo
+hg commit -m branch1
+
+cd ../repo1
+touch foo
+hg -q commit -A -m initial1
+echo 1 > foo
+hg commit -m 1
+cd ..
+
+hg -q clone -r 0 --pull -U repo0 repo0-upstream
+
+cat >> hgweb.conf << EOF
+[paths]
+/ = $TESTTMP/server/*
+EOF
+
+hg serve -d -p $HGPORT --pid-file hg.pid --web-conf hgweb.conf
+cat hg.pid >> $DAEMON_PIDS
+
+cd $TESTTMP
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-abandonded-transaction.t
@@ -0,0 +1,65 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ hg -R dest --config extensions.strip= strip -r aada1b3e573f --no-backup
+
+Simulate an abandonded transaction
+
+  $ touch $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg/store/journal
+
+Pulling when there is an abandoned transaction should automatically recover
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision aada1b3e573f
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain aada1b3e573f)
+  searching for changes
+  (abandoned transaction found; trying to recover)
+  rolling back interrupted transaction
+  (attempting checkout from beginning)
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain aada1b3e573f)
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files (+1 heads)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+Now simulate an abandoned transaction on an initial checkout
+
+  $ hg -R dest --config extensions.strip= strip -r aada1b3e573f --no-backup
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ touch $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg/store/journal
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest2 --revision aada1b3e573f
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest2
+  (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  searching for changes
+  (abandoned transaction found; trying to recover)
+  rolling back interrupted transaction
+  (attempting checkout from beginning)
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest2
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain aada1b3e573f)
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files (+1 heads)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-corrupt-repo.t
@@ -0,0 +1,75 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+  $ hg -R dest --config extensions.strip= strip -r aada1b3e573f --no-backup
+
+Corrupt the manifest
+
+  $ cat >> $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg/store/00manifest.i << EOF
+  > baddata
+  > EOF
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision aada1b3e573f
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain aada1b3e573f)
+  searching for changes
+  adding changesets
+  adding manifests
+  transaction abort!
+  rollback completed
+  (repo corruption: index 00manifest.i is corrupted; deleting shared store)
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (shared store does not exist; deleting)
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+Now check corruption is handled during clone
+
+  $ hg -R dest --config extensions.strip= strip -r aada1b3e573f --no-backup
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cat >> $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg/store/00manifest.i << EOF
+  > baddata
+  > EOF
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest1 --revision aada1b3e573f
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest1
+  (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  searching for changes
+  adding changesets
+  adding manifests
+  transaction abort!
+  rollback completed
+  (repo corruption: index 00manifest.i is corrupted; deleting shared store)
+  ensuring http://localhost:$HGPORT/repo0@aada1b3e573f is available at dest1
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (shared store does not exist; deleting)
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-dest-no-hg.t
@@ -0,0 +1,16 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Cloning to an existing directory that isn't a hg checkout will abort
+
+  $ mkdir dest
+  $ touch dest/file0
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision tip
+  ensuring http://localhost:$HGPORT/repo0@tip is available at dest
+  abort: destination exists but no .hg directory
+  [255]
+
+file0 should still be present
+
+  $ ls dest
+  file0
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-dest-shared-state.t
@@ -0,0 +1,70 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Checking out to an existing repo that isn't shared will blow it away
+
+  $ hg init dest0
+  $ touch dest0/file0
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest0 --revision tip
+  ensuring http://localhost:$HGPORT/repo0@tip is available at dest0
+  (destination is not shared; deleting)
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+  $ ls dest0
+  foo
+
+If shared path points nowhere, repo is "corrupt"; should be blown away
+
+  $ hg share -U dest0 missingsharepath
+  $ cat > missingsharepath/.hg/sharedpath << EOF
+  > does_not_exist
+  > EOF
+  $ touch missingsharepath/file0
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 missingsharepath --revision tip
+  ensuring http://localhost:$HGPORT/repo0@tip is available at missingsharepath
+  (existing repository shared store: does_not_exist)
+  (shared store does not exist; deleting)
+  (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+  $ ls missingsharepath
+  foo
+
+  $ cat missingsharepath/.hg/sharedpath
+  $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg (no-eol)
+
+If shared path does not point to pooled storage, it should get nuked as
+we require pooled storage
+
+  $ hg share -U dest0 nopoolshare
+  $ hg init fakeshare
+  $ cat > nopoolshare/.hg/sharedpath << EOF
+  > $TESTTMP/fakeshare/.hg
+  > EOF
+
+  $ touch nopoolshare/file0
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 nopoolshare --revision tip
+  ensuring http://localhost:$HGPORT/repo0@tip is available at nopoolshare
+  (existing repository shared store: $TESTTMP/fakeshare/.hg)
+  (shared store does not belong to pooled storage; deleting to improve efficiency)
+  (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+  $ ls nopoolshare
+  foo
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-missing-parent-dirs.t
@@ -0,0 +1,31 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Missing parent of destination directory will be created automatically
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 parent0/parent1/dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at parent0/parent1/dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+Missing parent of share pool directory will be created automatically
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b --sharebase shareparent/sharebase
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-purge.t
@@ -0,0 +1,69 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Not specifying --purge won't purge checkout
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ touch dest/file0
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ ls dest
+  file0
+  foo
+
+Specifying purge will delete files
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b --purge
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (purging working directory)
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ ls dest
+  foo
+
+Ignored files are also purged when requested
+
+  $ cd dest
+  $ cat > .hgignore << EOF
+  > .pyc$
+  > EOF
+  $ hg -q commit -A -m hgignore
+
+  $ touch foo.pyc
+  $ cd ..
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ ls dest
+  foo
+  foo.pyc
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b --purge
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (purging working directory)
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+  $ ls dest
+  foo
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-revision-branch.t
@@ -0,0 +1,58 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Must specify revision or branch argument
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest
+  abort: must specify one of --revision or --branch
+  [255]
+
+Only 1 of revision and branch can be specified
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --branch default --revision 5d6cdc75a09b
+  abort: cannot specify both --revision and --branch
+  [255]
+
+Specifying branch argument will checkout branch
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --branch default
+  ensuring http://localhost:$HGPORT/repo0@default is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  (pulling to obtain default)
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+Specifying branch argument will always attempt to pull because branch revisions can change
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --branch default
+  ensuring http://localhost:$HGPORT/repo0@default is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain default)
+  no changes found
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+Updating to another branch works
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --branch branch1
+  ensuring http://localhost:$HGPORT/repo0@branch1 is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain branch1)
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to aada1b3e573f7272bb2ef93b34acbf0f77c69d44
+
+Specifying revision will switch away from branch
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-unrelated.t
@@ -0,0 +1,53 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+Attempting to pull/checkout an unrelated repo will blow away the destination
+
+  $ touch dest/file0
+  $ hg robustcheckout http://localhost:$HGPORT/repo1 dest --revision 7d5b54cb09e1
+  ensuring http://localhost:$HGPORT/repo1@7d5b54cb09e1 is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (pulling to obtain 7d5b54cb09e1)
+  searching for changes
+  (repository is unrelated; deleting)
+  ensuring http://localhost:$HGPORT/repo1@7d5b54cb09e1 is available at dest
+  (sharing from new pooled repository 65cd4e3b46a3f22a08ec4162871e67f57c322f6a)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 1 files
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 7d5b54cb09e1172a3684402520112cab3f3a1b70
+
+  $ ls dest
+  foo
+
+And again for safe measure
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/65cd4e3b46a3f22a08ec4162871e67f57c322f6a/.hg)
+  (pulling to obtain 5d6cdc75a09b)
+  searching for changes
+  (repository is unrelated; deleting)
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from existing pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-update.t
@@ -0,0 +1,73 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 3 changesets with 3 changes to 1 files (+1 heads)
+  searching for changes
+  no changes found
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+Modifications to a tracked file should get lost during update
+
+  $ cat dest/foo
+  1
+
+  $ cat > dest/foo << EOF
+  > modified
+  > EOF
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ cat dest/foo
+  1
+
+Modifications should also get lost when updating to new revision
+
+  $ cat > dest/foo << EOF
+  > modified
+  > EOF
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision b8b78f0253d8
+  ensuring http://localhost:$HGPORT/repo0@b8b78f0253d8 is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to b8b78f0253d822e33ba652fd3d80a5c0837cfdf3
+
+  $ cat dest/foo
+
+Added and copied files will be lost during update
+
+  $ cd dest
+  $ touch bar
+  $ hg add bar
+  $ hg mv foo baz
+  $ cd ..
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ hg -R dest status
+  ? bar
+  ? baz
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --revision 5d6cdc75a09b --purge
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (existing repository shared store: $TESTTMP/share/b8b78f0253d822e33ba652fd3d80a5c0837cfdf3/.hg)
+  (purging working directory)
+  0 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce
+
+  $ hg -R dest status
new file mode 100644
--- /dev/null
+++ b/hgext/robustcheckout/tests/test-upstream.t
@@ -0,0 +1,23 @@
+  $ . $TESTDIR/hgext/robustcheckout/tests/helpers.sh
+
+Specifying an upstream repo will clone from it and pull from normal repo
+
+  $ hg robustcheckout http://localhost:$HGPORT/repo0 dest --upstream http://localhost:$HGPORT/repo0-upstream --revision 5d6cdc75a09b
+  ensuring http://localhost:$HGPORT/repo0@5d6cdc75a09b is available at dest
+  (cloning from upstream repo http://localhost:$HGPORT/repo0-upstream)
+  (sharing from new pooled repository b8b78f0253d822e33ba652fd3d80a5c0837cfdf3)
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  searching for changes
+  no changes found
+  (pulling to obtain 5d6cdc75a09b)
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  updated to 5d6cdc75a09bcccf76f9339a28e1d89360c59dce