Bug 1270951 - Be less strict about auto pooled shared repos; r?jlund
mozharness was updated in
bug 1270951 to always create and ensure auto
pooled storage. What this means is that .hg/sharedpath always points to
a path of the form "<sha1>/.hg" where <sha1> is a 40 character
hexidecimal changeset revision corresponding to the logical repo the
share is from.
What I didn't realize at the time I landed that was that this conflicts
with hgtool.
hgtool has code for verifying that .hg/sharedpath points to the base name
of the repository we're cloning from. In reality, this will likely never
match a 40 character hexidecimal changeset and hgtool will blow away the
working directory. This means that mozharness and hgtool will each step
on each other's toes, deleting working directories the other created.
This wouldn't be a problem if mozharness were the only game in town.
However, the new mozharness isn't deployed everywhere yet, so hgtool
will still run and blow away mozharness's auto pooled working directory.
The solution to this problem as implemented in this commit is to teach
hgtool just enough about auto pooled shared repos so it doesn't
interfere with mozharness-created repos. We add logic to look for
.hg/sharedpath pointing to a <sha1>/.hg path. If found, we assume
pooled storage and don't engage the strict share checking already
built into hgtool.
We could potentially remove the strict share checking in hgtool.
However, I don't want to do this out of an abundance of caution.
I'm not sure what all is still using hgtool and I don't want to take
a risk that changing hgtool's share behavior in the absence of
mozharness will lead to catastrophe.
MozReview-Commit-ID: 66y8kLrgor
--- a/lib/python/mozilla_buildtools/test/test_util_hg.py
+++ b/lib/python/mozilla_buildtools/test/test_util_hg.py
@@ -1197,8 +1197,62 @@ class TestHg(unittest.TestCase):
get_hg_output(["status"])
self.fail("CalledProcessError not raised")
except subprocess.CalledProcessError:
self.assertTrue(mocked_log.exception.called, "log.exception not called")
def testHasRev(self):
self.assertTrue(has_rev(self.repodir, self.revisions[0]))
self.assertFalse(has_rev(self.repodir, self.revisions[0] + 'g'))
+
+ def testMercurialPooledMissingStore(self):
+ clone(self.repodir, self.wc)
+ shared_path = os.path.join(self.wc, '.hg', 'sharedpath')
+ with open(shared_path, 'wb') as fh:
+ sha1 = 'a' * 40
+ fh.write('/does/not/exist/%s/.hg' % sha1)
+
+ mercurial(self.repodir, self.wc)
+
+ self.assertFalse(os.path.exists(shared_path))
+
+ def _make_pooled_share(self):
+ rev0 = get_hg_output(['log', '-r', '0', '-T', '{node}'], cwd=self.repodir)
+
+ share_base = os.path.join(self.tmpdir, 'share')
+ shutil.copytree(self.repodir, os.path.join(share_base, rev0))
+
+ init(self.wc)
+ with open(os.path.join(self.wc, '.hg', 'sharedpath'), 'wb') as fh:
+ fh.write(os.path.join(share_base, rev0, '.hg'))
+ shutil.rmtree(os.path.join(self.wc, '.hg', 'store'))
+
+ def testMercurialPooledPurge(self):
+ self._make_pooled_share()
+
+ extra_path = os.path.join(self.wc, 'to_delete')
+ with open(extra_path, 'a'):
+ pass
+
+ mercurial(self.repodir, self.wc, autoPurge=True)
+
+ self.assertFalse(os.path.exists(extra_path))
+
+ def testMercurialPooledUpdateExisting(self):
+ self._make_pooled_share()
+
+ rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1])
+ self.assertEquals(rev, self.revisions[-1])
+
+ wrev = get_hg_output(['log', '-r', '.', '-T', '{node|short}'], cwd=self.wc)
+ self.assertEqual(rev, wrev)
+
+ def testMercurialPooledPullUnrelated(self):
+ self._make_pooled_share()
+
+ repo2 = os.path.join(self.tmpdir, 'repo2')
+ run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2],
+ env={'extra': 'pooled_pull_unrelated'})
+
+ self.assertNotEqual(self.revisions, getRevisions(repo2))
+
+ mercurial(repo2, self.wc)
+ self.assertEqual(getRevisions(self.wc), getRevisions(repo2))
--- a/lib/python/util/hg.py
+++ b/lib/python/util/hg.py
@@ -482,16 +482,63 @@ def mercurial(repo, dest, branch=None, r
dest = os.path.abspath(dest)
if shareBase is DefaultShareBase:
shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
log.info("Reporting hg version in use")
cmd = ['hg', '-q', 'version']
run_cmd(cmd, cwd='.')
+ # If the working directory points to a share from auto pooled storage
+ # (identified by store paths of form <sha1>/.hg) it was created via
+ # modern mechanisms (likely mozharness) and isn't bound by the same
+ # strict requirements implemented below. We go through a specialized
+ # code path that mimics the mozharness behavior.
+ dest_sharedpath = os.path.join(dest, '.hg', 'sharedpath')
+ if os.path.exists(dest_sharedpath):
+ with open(dest_sharedpath, 'rb') as fh:
+ store_path = fh.read().strip()
+
+ if re.search('[a-f0-9]{40}/\.hg$', store_path.replace('\\', '/')):
+ log.info('pooled storage detected')
+ if not os.path.exists(store_path):
+ log.warning('store path %s does not exist; clobbering', store_path)
+ remove_path(dest)
+ return mercurial(repo, dest,
+ branch=branch,
+ revision=revision,
+ update_dest=update_dest,
+ shareBase=shareBase,
+ allowUnsharedLocalClones=allowUnsharedLocalClones,
+ clone_by_rev=clone_by_rev,
+ mirrors=mirrors,
+ bundles=bundles,
+ autoPurge=autoPurge)
+
+ try:
+ if autoPurge:
+ purge(dest)
+
+ if revision and is_hg_cset(revision) and has_rev(dest, revision):
+ log.info('skipping pull since we already have %s', revision)
+ if update_dest:
+ return update(dest, branch=branch, revision=revision)
+
+ return revision
+
+ return pull(repo, dest, update_dest=update_dest, branch=branch,
+ revision=revision, mirrors=mirrors)
+ except subprocess.CalledProcessError:
+ # This will undo our intelligent auto pooled storage. Hopefully
+ # this never happens. If it does, we can make the logic more
+ # robust.
+ log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
+ log.debug("Exception: ", exc_info=True)
+ remove_path(dest)
+
if shareBase:
# Check that 'hg share' works
try:
log.info("Checking if share extension works")
output = get_hg_output(['help', 'share'], dont_log=True)
if 'no commands defined' in output:
# Share extension is enabled, but not functional
log.info("Disabling sharing since share extension doesn't seem to work (1)")