--- a/testing/mozharness/external_tools/robustcheckout.py
+++ b/testing/mozharness/external_tools/robustcheckout.py
@@ -13,37 +13,54 @@ from __future__ import absolute_import
import contextlib
import errno
import functools
import os
import random
import re
import socket
+import ssl
import time
import urllib2
from mercurial.i18n import _
from mercurial.node import hex
from mercurial import (
commands,
error,
exchange,
extensions,
cmdutil,
hg,
+ registrar,
scmutil,
util,
)
-testedwith = '3.7 3.8 3.9 4.0 4.1'
+testedwith = '3.7 3.8 3.9 4.0 4.1 4.2 4.3'
minimumhgversion = '3.7'
cmdtable = {}
-command = cmdutil.command(cmdtable)
+
+# Mercurial 4.3 introduced registrar.command as a replacement for
+# cmdutil.command.
+if util.safehasattr(registrar, 'command'):
+ command = registrar.command(cmdtable)
+else:
+ command = cmdutil.command(cmdtable)
+
+# Mercurial 4.2 introduced the vfs module and deprecated the symbol in
+# scmutil.
+def getvfs():
+ try:
+ from mercurial.vfs import vfs
+ return vfs
+ except ImportError:
+ return scmutil.vfs
if os.name == 'nt':
import ctypes
# Get a reference to the DeleteFileW function
# DeleteFileW accepts filenames encoded as a null terminated sequence of
# wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
@@ -171,50 +188,73 @@ def _docheckout(ui, url, dest, upstream,
def callself():
return _docheckout(ui, url, dest, upstream, revision, branch, purge,
sharebase, networkattemptlimit, networkattempts)
ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
dest))
- destvfs = scmutil.vfs(dest, audit=False, realpath=True)
+ # We assume that we're the only process on the machine touching the
+ # repository paths that we were told to use. This means our recovery
+ # scenario when things aren't "right" is to just nuke things and start
+ # from scratch. This is easier to implement than verifying the state
+ # of the data and attempting recovery. And in some scenarios (such as
+ # potential repo corruption), it is probably faster, since verifying
+ # repos can take a while.
+
+ destvfs = getvfs()(dest, audit=False, realpath=True)
+
+ def deletesharedstore(path=None):
+ storepath = path or destvfs.read('.hg/sharedpath').strip()
+ if storepath.endswith('.hg'):
+ storepath = os.path.dirname(storepath)
+
+ storevfs = getvfs()(storepath, audit=False)
+ storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists('.hg'):
raise error.Abort('destination exists but no .hg directory')
# Require checkouts to be tied to shared storage because efficiency.
if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
ui.warn('(destination is not shared; deleting)\n')
destvfs.rmtree(forcibly=True)
# Verify the shared path exists and is using modern pooled storage.
if destvfs.exists('.hg/sharedpath'):
storepath = destvfs.read('.hg/sharedpath').strip()
ui.write('(existing repository shared store: %s)\n' % storepath)
if not os.path.exists(storepath):
- ui.warn('(shared store does not exist; deleting)\n')
+ ui.warn('(shared store does not exist; deleting destination)\n')
destvfs.rmtree(forcibly=True)
elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
ui.warn('(shared store does not belong to pooled storage; '
- 'deleting to improve efficiency)\n')
+ 'deleting destination to improve efficiency)\n')
destvfs.rmtree(forcibly=True)
+ storevfs = getvfs()(storepath, audit=False)
+ if storevfs.isfileorlink('store/lock'):
+ ui.warn('(shared store has an active lock; assuming it is left '
+ 'over from a previous process and that the store is '
+ 'corrupt; deleting store and destination just to be '
+ 'sure)\n')
+ destvfs.rmtree(forcibly=True)
+ deletesharedstore(storepath)
+
# FUTURE when we require generaldelta, this is where we can check
# for that.
- def deletesharedstore():
- storepath = destvfs.read('.hg/sharedpath').strip()
- if storepath.endswith('.hg'):
- storepath = os.path.dirname(storepath)
-
- storevfs = scmutil.vfs(storepath, audit=False)
- storevfs.rmtree(forcibly=True)
+ if destvfs.isfileorlink('.hg/wlock'):
+ ui.warn('(dest has an active working directory lock; assuming it is '
+ 'left over from a previous process and that the destination '
+ 'is corrupt; deleting it just to be sure)\n')
+ destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if e.message == _('abandoned transaction found'):
ui.warn('(abandoned transaction found; trying to recover)\n')
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn('(could not recover repo state; '
'deleting shared store)\n')
@@ -262,16 +302,23 @@ def _docheckout(ui, url, dest, upstream,
ui.warn('(repository is unrelated; deleting)\n')
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_('stream ended unexpectedly')):
ui.warn('%s\n' % e.args[0])
# Will raise if failure limit reached.
handlenetworkfailure()
return True
+ elif isinstance(e, ssl.SSLError):
+ # Assume all SSL errors are due to the network, as Mercurial
+ # should convert non-transport errors like cert validation failures
+ # to error.Abort.
+ ui.warn('ssl error: %s\n' % e)
+ handlenetworkfailure()
+ return True
elif isinstance(e, urllib2.URLError):
if isinstance(e.reason, socket.error):
ui.warn('socket error: %s\n' % e.reason)
handlenetworkfailure()
return True
return False
@@ -290,17 +337,17 @@ def _docheckout(ui, url, dest, upstream,
if upstream:
ui.write('(cloning from upstream repo %s)\n' % upstream)
cloneurl = upstream or url
try:
res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
shareopts={'pool': sharebase, 'mode': 'identity'})
- except (error.Abort, urllib2.URLError) as e:
+ except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()
@@ -349,17 +396,17 @@ def _docheckout(ui, url, dest, upstream,
(branch, checkoutrevision))
if checkoutrevision in repo:
ui.warn('(revision already present locally; not pulling)\n')
else:
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort('unable to pull requested revision')
- except (error.Abort, urllib2.URLError) as e:
+ except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
deletesharedstore()