--- a/testing/mozharness/mozharness/base/vcs/mercurial.py
+++ b/testing/mozharness/mozharness/base/vcs/mercurial.py
@@ -1,43 +1,60 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Mercurial VCS support.
-
-Largely copied/ported from
-https://hg.mozilla.org/build/tools/file/cf265ea8fb5e/lib/python/util/hg.py .
"""
import os
import re
import subprocess
from collections import namedtuple
from urlparse import urlsplit
import sys
sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
+import mozharness
from mozharness.base.errors import HgErrorList, VCSException
-from mozharness.base.log import LogMixin
+from mozharness.base.log import LogMixin, OutputParser
from mozharness.base.script import ScriptMixin
from mozharness.base.transfer import TransferMixin
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+
HG_OPTIONS = ['--config', 'ui.merge=internal:merge']
# MercurialVCS {{{1
# TODO Make the remaining functions more mozharness-friendly.
# TODO Add the various tag functionality that are currently in
# build/tools/scripts to MercurialVCS -- generic tagging logic belongs here.
REVISION, BRANCH = 0, 1
+class RepositoryUpdateRevisionParser(OutputParser):
+ """Parse `hg pull` output for "repository unrelated" errors."""
+ revision = None
+ RE_UPDATED = re.compile('^updated to ([a-f0-9]{40})$')
+
+ def parse_single_line(self, line):
+ m = self.RE_UPDATED.match(line)
+ if m:
+ self.revision = m.group(1)
+
+ return super(RepositoryUpdateRevisionParser, self).parse_single_line(line)
+
+
def make_hg_url(hg_host, repo_path, protocol='http', revision=None,
filename=None):
"""Helper function.
Construct a valid hg url from a base hg url (hg.mozilla.org),
repo_path, revision and possible filename
"""
base = '%s://%s' % (protocol, hg_host)
@@ -300,184 +317,81 @@ class MercurialVCS(ScriptMixin, LogMixin
cmd.append('--new-branch')
cmd.append(remote)
status = self.run_command(cmd, cwd=src, error_list=HgErrorList, success_codes=(0, 1),
return_type="num_errors")
if status:
raise VCSException("Can't push %s to %s!" % (src, remote))
return status
- # hg share methods {{{2
- def query_can_share(self):
- if self.can_share is not None:
- return self.can_share
- # Check that 'hg share' works
- self.can_share = True
- try:
- self.info("Checking if share extension works.")
- output = self.get_output_from_command(self.hg + ['help', 'share'],
- silent=True,
- throw_exception=True)
- if 'no commands defined' in output:
- # Share extension is enabled, but not functional
- self.warning("Disabling sharing since share extension doesn't seem to work (1)")
- self.can_share = False
- elif 'unknown command' in output or 'hg help extensions' in output:
- # Share extension is disabled
- self.warning("Disabling sharing since share extension doesn't seem to work (2)")
- self.can_share = False
- except subprocess.CalledProcessError:
- # The command failed, so disable sharing
- self.warning("Disabling sharing since share extension doesn't seem to work (3)")
- self.can_share = False
- if self.can_share:
- self.info("hg share works.")
- return self.can_share
-
- def _ensure_shared_repo_and_revision(self, share_base):
- """The shared dir logic is complex enough to warrant its own
- helper method.
-
- If allow_unshared_local_clones is True and we're trying to use the
- share extension but fail, then we will be able to clone from the
- shared repo to our destination. If this is False, the default, the
- if we don't have the share extension we will just clone from the
- remote repository.
- """
- c = self.vcs_config
- dest = os.path.abspath(c['dest'])
- repo = c['repo']
- revision = c.get('revision')
- branch = c.get('branch')
- if not self.query_can_share():
- raise VCSException("%s called when sharing is not allowed!" % __name__)
-
- # If the working directory already exists and isn't using share
- # when we want to use share, clobber.
- #
- # The original util.hg.mercurial() tried to pull repo into dest
- # instead. That can help if the share extension fails.
- # But it can also result in pulling a different repo B into an
- # existing clone of repo A, which may have unexpected results.
- if os.path.exists(dest):
- sppath = os.path.join(dest, ".hg", "sharedpath")
- if not os.path.exists(sppath):
- self.info("No file %s; removing %s." % (sppath, dest))
- self.rmtree(dest)
- if not os.path.exists(os.path.dirname(dest)):
- self.mkdir_p(os.path.dirname(dest))
- shared_repo = os.path.join(share_base, self.get_repo_path(repo))
- dest_shared_path = os.path.join(dest, '.hg', 'sharedpath')
- if os.path.exists(dest_shared_path):
- # Make sure that the sharedpath points to shared_repo
- dest_shared_path_data = os.path.realpath(open(dest_shared_path).read())
- norm_shared_repo = os.path.realpath(os.path.join(shared_repo, '.hg'))
- if dest_shared_path_data != norm_shared_repo:
- # Clobber!
- self.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering" % (dest_shared_path_data, repo, norm_shared_repo))
- self.rmtree(dest)
-
- self.info("Updating shared repo")
- if os.path.exists(shared_repo):
- try:
- self.pull(repo, shared_repo)
- except VCSException:
- self.warning("Error pulling changes into %s from %s; clobbering" % (shared_repo, repo))
- self.exception(level='debug')
- self.clone(repo, shared_repo)
- else:
- self.clone(repo, shared_repo)
-
- if os.path.exists(dest):
- try:
- self.pull(shared_repo, dest)
- status = self.update(dest, branch=branch, revision=revision)
- return status
- except VCSException:
- self.rmtree(dest)
- try:
- self.info("Trying to share %s to %s" % (shared_repo, dest))
- return self.share(shared_repo, dest, branch=branch, revision=revision)
- except VCSException:
- if not c.get('allow_unshared_local_clones'):
- # Re-raise the exception so it gets caught below.
- # We'll then clobber dest, and clone from original
- # repo
- raise
-
- self.warning("Error calling hg share from %s to %s; falling back to normal clone from shared repo" % (shared_repo, dest))
- # Do a full local clone first, and then update to the
- # revision we want
- # This lets us use hardlinks for the local clone if the
- # OS supports it
- try:
- self.clone(shared_repo, dest, update_dest=False)
- return self.update(dest, branch=branch, revision=revision)
- except VCSException:
- # Need better fallback
- self.error("Error updating %s from shared_repo (%s): " % (dest, shared_repo))
- self.exception(level='error')
- self.rmtree(dest)
-
- def share(self, source, dest, branch=None, revision=None):
- """Creates a new working directory in "dest" that shares history
- with "source" using Mercurial's share extension
- """
- self.info("Sharing %s to %s." % (source, dest))
- self.mkdir_p(dest)
- if self.run_command(self.hg + ['share', '-U', source, dest],
- error_list=HgErrorList):
- raise VCSException("Unable to share %s to %s!" % (source, dest))
- return self.update(dest, branch=branch, revision=revision)
-
- # End hg share methods 2}}}
+ @property
+ def robustcheckout_path(self):
+ """Path to the robustcheckout extension."""
+ ext = os.path.join(external_tools_path, 'robustcheckout.py')
+ if os.path.exists(ext):
+ return ext
def ensure_repo_and_revision(self):
"""Makes sure that `dest` is has `revision` or `branch` checked out
from `repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
"""
c = self.vcs_config
- for conf_item in ('dest', 'repo'):
- assert self.vcs_config[conf_item]
- dest = os.path.abspath(c['dest'])
- repo = c['repo']
- revision = c.get('revision')
+ dest = c['dest']
+ repo_url = c['repo']
+ rev = c.get('revision')
branch = c.get('branch')
- share_base = c.get('vcs_share_base',
- os.environ.get("HG_SHARE_BASE_DIR", None))
- msg = "Setting %s to %s" % (dest, repo)
+ purge = c.get('clone_with_purge', False)
+ upstream = c.get('clone_upstream_url')
+
+ # The API here is kind of bad because we're relying on state in
+ # self.vcs_config instead of passing arguments. This confuses
+ # scripts that have multiple repos. This includes the clone_tools()
+ # step :(
+
+ if not rev and not branch:
+ self.warning('did not specify revision or branch; assuming "default"')
+ branch = 'default'
+
+ share_base = c.get('vcs_share_base', os.environ.get('HG_SHARE_BASE_DIR', None))
+
+ # We require shared storage is configured because it guarantees we
+ # only have 1 local copy of logical repo stores.
+ if not share_base:
+ raise VCSException('vcs share base not defined; '
+ 'refusing to operate sub-optimally')
+
+ if not self.robustcheckout_path:
+ raise VCSException('could not find the robustcheckout Mercurial extension')
+
+ args = self.hg + [
+ '--config', 'extensions.robustcheckout=%s' % self.robustcheckout_path,
+ 'robustcheckout', repo_url, dest, '--sharebase', share_base,
+ ]
+ if purge:
+ args.append('--purge')
+ if upstream:
+ args.extend(['--upstream', upstream])
+
+ if rev:
+ args.extend(['--revision', rev])
if branch:
- msg += " on branch %s" % branch
- if revision:
- msg += " revision %s" % revision
- if share_base:
- msg += " using shared directory %s" % share_base
- self.info("%s." % msg)
- if share_base and not self.query_can_share():
- share_base = None
-
- if share_base:
- return self._ensure_shared_repo_and_revision(share_base)
+ args.extend(['--branch', branch])
- # Non-shared
- if os.path.exists(dest):
- try:
- self.pull(repo, dest)
- return self.update(dest, branch=branch, revision=revision)
- except VCSException:
- self.warning("Error pulling changes into %s from %s; clobbering" % (dest, repo))
- self.exception(level='debug')
- self.rmtree(dest)
- elif not os.path.exists(os.path.dirname(dest)):
- self.mkdir_p(os.path.dirname(dest))
- self.clone(repo, dest)
- return self.update(dest, branch=branch, revision=revision)
+ parser = RepositoryUpdateRevisionParser(config=self.config,
+ log_obj=self.log_obj)
+ if self.run_command(args, output_parser=parser):
+ raise VCSException('repo checkout failed!')
+
+ if not parser.revision:
+ raise VCSException('could not identify revision updated to')
+
+ return parser.revision
def apply_and_push(self, localrepo, remote, changer, max_attempts=10,
ssh_username=None, ssh_key=None):
"""This function calls `changer' to make changes to the repo, and
tries its hardest to get them to the origin repo. `changer' must be
a callable object that receives two arguments: the directory of the
local repository, and the attempt number. This function will push
ALL changesets missing from remote.
--- a/testing/mozharness/test/test_base_vcs_mercurial.py
+++ b/testing/mozharness/test/test_base_vcs_mercurial.py
@@ -208,61 +208,16 @@ class TestHg(unittest.TestCase):
# Clone the original repo
m.clone(self.repodir, self.wc, update_dest=False)
# Hide the wanted error
m.config = {'log_to_console': False}
# Try and pull in changes from the new repo
self.assertRaises(mercurial.VCSException, m.pull, repo2, self.wc, update_dest=False)
- def test_share_unrelated(self):
- m = get_mercurial_vcs_obj()
- # Create a new repo
- repo2 = os.path.join(self.tmpdir, 'repo2')
- self._init_hg_repo(m, repo2)
-
- self.assertNotEqual(self.revisions, get_revisions(repo2))
-
- share_base = os.path.join(self.tmpdir, 'share')
-
- # Clone the original repo
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'vcs_share_base': share_base}
- m.ensure_repo_and_revision()
-
- # Clone the new repo
- m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': repo2, 'dest': self.wc, 'vcs_share_base': share_base}
- m.ensure_repo_and_revision()
-
- self.assertEquals(get_revisions(self.wc), get_revisions(repo2))
-
- def test_share_reset(self):
- m = get_mercurial_vcs_obj()
- share_base = os.path.join(self.tmpdir, 'share')
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'vcs_share_base': share_base}
-
- # Clone the original repo
- m.ensure_repo_and_revision()
-
- old_revs = self.revisions[:]
-
- # Reset the repo
- self._init_hg_repo(m, self.repodir)
-
- self.assertNotEqual(old_revs, get_revisions(self.repodir))
-
- # Try and update our working copy
- m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'vcs_share_base': share_base}
- m.config = {'log_to_console': False}
- m.ensure_repo_and_revision()
-
- self.assertEquals(get_revisions(self.repodir), get_revisions(self.wc))
- self.assertNotEqual(old_revs, get_revisions(self.wc))
-
def test_push(self):
m = get_mercurial_vcs_obj()
m.clone(self.repodir, self.wc, revision=self.revisions[-2])
m.push(src=self.repodir, remote=self.wc)
self.assertEquals(get_revisions(self.wc), self.revisions)
def test_push_with_branch(self):
m = get_mercurial_vcs_obj()
@@ -275,149 +230,110 @@ class TestHg(unittest.TestCase):
def test_push_with_revision(self):
m = get_mercurial_vcs_obj()
m.clone(self.repodir, self.wc, revision=self.revisions[-2])
m.push(src=self.repodir, remote=self.wc, revision=self.revisions[-1])
self.assertEquals(get_revisions(self.wc), self.revisions[-2:])
def test_mercurial(self):
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc}
+ m.vcs_config = {
+ 'repo': self.repodir,
+ 'dest': self.wc,
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
m.ensure_repo_and_revision()
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[0])
def test_push_new_branches_not_allowed(self):
m = get_mercurial_vcs_obj()
m.clone(self.repodir, self.wc, revision=self.revisions[0])
# Hide the wanted error
m.config = {'log_to_console': False}
self.assertRaises(Exception, m.push, self.repodir, self.wc, push_new_branches=False)
- def test_mercurial_with_new_share(self):
- m = get_mercurial_vcs_obj()
- share_base = os.path.join(self.tmpdir, 'share')
- sharerepo = os.path.join(share_base, self.repodir.lstrip("/"))
- os.mkdir(share_base)
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'vcs_share_base': share_base}
- m.ensure_repo_and_revision()
- self.assertEquals(get_revisions(self.repodir), get_revisions(self.wc))
- self.assertEquals(get_revisions(self.repodir), get_revisions(sharerepo))
-
- def test_mercurial_with_share_base_in_env(self):
- share_base = os.path.join(self.tmpdir, 'share')
- sharerepo = os.path.join(share_base, self.repodir.lstrip("/"))
- os.mkdir(share_base)
- try:
- os.environ['HG_SHARE_BASE_DIR'] = share_base
- m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc}
- m.ensure_repo_and_revision()
- self.assertEquals(get_revisions(self.repodir), get_revisions(self.wc))
- self.assertEquals(get_revisions(self.repodir), get_revisions(sharerepo))
- finally:
- del os.environ['HG_SHARE_BASE_DIR']
-
- def test_mercurial_with_existing_share(self):
- m = get_mercurial_vcs_obj()
- share_base = os.path.join(self.tmpdir, 'share')
- sharerepo = os.path.join(share_base, self.repodir.lstrip("/"))
- os.mkdir(share_base)
- m.vcs_config = {'repo': self.repodir, 'dest': sharerepo}
- m.ensure_repo_and_revision()
- open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
- m.run_command(HG + ['add', 'test.txt'], cwd=self.repodir)
- m.run_command(HG + ['commit', '-m', 'adding changeset'], cwd=self.repodir)
- m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'vcs_share_base': share_base}
- m.ensure_repo_and_revision()
- self.assertEquals(get_revisions(self.repodir), get_revisions(self.wc))
- self.assertEquals(get_revisions(self.repodir), get_revisions(sharerepo))
-
def test_mercurial_relative_dir(self):
m = get_mercurial_vcs_obj()
repo = os.path.basename(self.repodir)
wc = os.path.basename(self.wc)
- m.vcs_config = {'repo': repo, 'dest': wc, 'revision': self.revisions[-1]}
+ m.vcs_config = {
+ 'repo': repo,
+ 'dest': wc,
+ 'revision': self.revisions[-1],
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
m.chdir(os.path.dirname(self.repodir))
try:
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[-1])
m.info("Creating test.txt")
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': repo, 'dest': wc, 'revision': self.revisions[0]}
+ m.vcs_config = {
+ 'repo': repo,
+ 'dest': wc,
+ 'revision': self.revisions[0],
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
finally:
m.chdir(self.pwd)
def test_mercurial_update_tip(self):
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'revision': self.revisions[-1]}
+ m.vcs_config = {
+ 'repo': self.repodir,
+ 'dest': self.wc,
+ 'revision': self.revisions[-1],
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc}
+ m.vcs_config = {
+ 'repo': self.repodir,
+ 'dest': self.wc,
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
def test_mercurial_update_rev(self):
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'revision': self.revisions[-1]}
+ m.vcs_config = {
+ 'repo': self.repodir,
+ 'dest': self.wc,
+ 'revision': self.revisions[-1],
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
m = get_mercurial_vcs_obj()
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc, 'revision': self.revisions[0]}
+ m.vcs_config = {
+ 'repo': self.repodir,
+ 'dest': self.wc,
+ 'revision': self.revisions[0],
+ 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
+ }
rev = m.ensure_repo_and_revision()
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
- # TODO: this test doesn't seem to be compatible with mercurial()'s
- # share() usage, and fails when HG_SHARE_BASE_DIR is set
- def test_mercurial_change_repo(self):
- # Create a new repo
- old_env = os.environ.copy()
- if 'HG_SHARE_BASE_DIR' in os.environ:
- del os.environ['HG_SHARE_BASE_DIR']
-
- m = get_mercurial_vcs_obj()
- try:
- repo2 = os.path.join(self.tmpdir, 'repo2')
- self._init_hg_repo(m, repo2)
-
- self.assertNotEqual(self.revisions, get_revisions(repo2))
-
- # Clone the original repo
- m.vcs_config = {'repo': self.repodir, 'dest': self.wc}
- m.ensure_repo_and_revision()
- self.assertEquals(get_revisions(self.wc), self.revisions)
- open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
-
- # Clone the new one
- m.vcs_config = {'repo': repo2, 'dest': self.wc}
- m.config = {'log_to_console': False}
- m.ensure_repo_and_revision()
- self.assertEquals(get_revisions(self.wc), get_revisions(repo2))
- # Make sure our local file went away
- self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt')))
- finally:
- os.environ.clear()
- os.environ.update(old_env)
-
def test_make_hg_url(self):
#construct an hg url specific to revision, branch and filename and try to pull it down
file_url = mercurial.make_hg_url(
"hg.mozilla.org",
'//build/tools/',
revision='FIREFOX_3_6_12_RELEASE',
filename="/lib/python/util/hg.py",
protocol='https',
@@ -448,39 +364,16 @@ class TestHg(unittest.TestCase):
repo_url = mercurial.make_hg_url(
"hg.mozilla.org",
"/build/tools",
protocol='ssh',
)
expected_url = "ssh://hg.mozilla.org/build/tools"
self.assertEquals(repo_url, expected_url)
- def test_share_repo(self):
- m = get_mercurial_vcs_obj()
- repo3 = os.path.join(self.tmpdir, 'repo3')
- m.share(self.repodir, repo3)
- # make sure shared history is identical
- self.assertEquals(self.revisions, get_revisions(repo3))
-
- def test_mercurial_share_outgoing(self):
- m = get_mercurial_vcs_obj()
- # ensure that outgoing changesets in a shared clone affect the shared history
- repo5 = os.path.join(self.tmpdir, 'repo5')
- repo6 = os.path.join(self.tmpdir, 'repo6')
- m.vcs_config = {'repo': self.repodir, 'dest': repo5}
- m.ensure_repo_and_revision()
- m.share(repo5, repo6)
- open(os.path.join(repo6, 'test.txt'), 'w').write("hello!")
- # modify the history of the new clone
- m.run_command(HG + ['add', 'test.txt'], cwd=repo6)
- m.run_command(HG + ['commit', '-m', 'adding changeset'], cwd=repo6)
- self.assertNotEquals(self.revisions, get_revisions(repo6))
- self.assertNotEquals(self.revisions, get_revisions(repo5))
- self.assertEquals(get_revisions(repo5), get_revisions(repo6))
-
def test_apply_and_push(self):
m = get_mercurial_vcs_obj()
m.clone(self.repodir, self.wc)
def c(repo, attempt):
m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
m.apply_and_push(self.wc, self.repodir, c)
self.assertEquals(get_revisions(self.wc), get_revisions(self.repodir))