Bug 1234913 - Part 2: Support git in |mach artifact install|. r?chmanchester
MozReview-Commit-ID: LL6kO8QS5p9
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -675,21 +675,27 @@ class ArtifactCache(CacheManager):
self.log(logging.INFO, 'artifact',
{'filename': result + PROCESSED_SUFFIX},
'Last installed binaries from local processed file {filename}')
class Artifacts(object):
'''Maintain state to efficiently fetch build artifacts from a Firefox tree.'''
- def __init__(self, tree, job=None, log=None, cache_dir='.', hg='hg', skip_cache=False):
+ def __init__(self, tree, job=None, log=None, cache_dir='.', python=None, hg=None, git=None, skip_cache=False):
self._tree = tree
self._job = job or self._guess_artifact_job()
self._log = log
+ if not python:
+ raise ValueError("Must provide path to Python")
+ if (hg and git) or (not hg and not git):
+ raise ValueError("Must provide path to exactly one of hg and git")
+ self._python = python
self._hg = hg
+ self._git = git
self._cache_dir = cache_dir
self._skip_cache = skip_cache
try:
self._artifact_job = get_job_details(self._job, log=self._log)
except KeyError:
self.log(logging.INFO, 'artifact',
{'job': self._job},
@@ -770,16 +776,58 @@ class Artifacts(object):
yield rev_info[0], tuple(rev_info[1:])
if not count:
raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n\n'
'Try running |hg pushlogsync|;\n'
'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format(
num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT))
+ def _find_git_pushheads(self, rev):
+ """Return an iterator of (hg_hash, {tree-set}) associating hg revision
+ hashes that might be pushheads with the trees they are known
+ to be in.
+
+ More recent hashes should come earlier in the list. It's okay
+ for tree-set to be the empty set {}; in that case, we'll query
+ the TaskCluster Index to determine the tree-set.
+ """
+
+ import which
+ cinnabar = which.which('git-cinnabar')
+
+ # First commit is HEAD, next is HEAD~1, etc.
+ rev_list = subprocess.check_output([
+ self._git, 'rev-list', '--ancestry-path',
+ 'HEAD~{num}..HEAD'.format(num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT),
+ ])
+
+ hg_hash_list = subprocess.check_output([
+ self._python, cinnabar, 'git2hg',
+ ] + rev_list.splitlines())
+
+ zeroes = "0" * 40
+ self._index = taskcluster.Index()
+
+ # We don't have pushlog data, so we never know there is a push
+ # (resulting in any kind of job) corresponding to any trees.
+ trees = tuple()
+ count = 0
+ for hg_hash in hg_hash_list.splitlines():
+ hg_hash = hg_hash.strip()
+ if not hg_hash or hg_hash == zeroes:
+ continue
+ count += 1
+ yield (hg_hash, trees)
+
+ if not count:
+ raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n\n'
+ 'See https://developer.mozilla.org/en-US/docs/Artifact_builds'.format(
+ num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT))
+
def find_pushhead_artifacts(self, task_cache, tree_cache, job, pushhead, trees):
known_trees = set(tree_cache.artifact_trees(pushhead, trees))
if not known_trees:
return None
if not trees:
# Accept artifacts from any tree where they are available.
trees = list(known_trees)
trees.sort()
@@ -886,16 +934,20 @@ class Artifacts(object):
{'count': count},
'Tried {count} pushheads, no built artifacts found.')
return 1
def install_from_hg_recent(self, distdir):
hg_pushheads = self._find_hg_pushheads()
return self._install_from_hg_pushheads(hg_pushheads, distdir)
+ def install_from_git_recent(self, distdir):
+ hg_pushheads = self._find_git_pushheads('HEAD')
+ return self._install_from_hg_pushheads(hg_pushheads, distdir)
+
def install_from_hg_revset(self, revset, distdir):
revision = subprocess.check_output([self._hg, 'log', '--template', '{node}\n',
'-r', revset]).strip()
if len(revision.split('\n')) != 1:
raise ValueError('hg revision specification must resolve to exactly one commit')
hg_pushheads = [(revision, tuple())]
self.log(logging.INFO, 'artifact',
{'revset': revset,
@@ -911,18 +963,24 @@ class Artifacts(object):
return self.install_from_file(source, distdir)
elif source and urlparse.urlparse(source).scheme:
return self.install_from_url(source, distdir)
else:
if source is None and 'MOZ_ARTIFACT_REVISION' in os.environ:
source = os.environ['MOZ_ARTIFACT_REVISION']
if source:
+ if self._git:
+ # TODO: Handle MOZ_ARTIFACT_{HG,GIT}_REVISION.
+ raise ValueError("MOZ_ARTIFACT_REVISION is not yet supported with git")
return self.install_from_hg_revset(source, distdir)
+ if self._git:
+ return self.install_from_git_recent(distdir)
+
return self.install_from_hg_recent(distdir)
def print_last(self):
self.log(logging.INFO, 'artifact',
{},
'Printing last used artifact details.')
self._tree_cache.print_last()
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1447,20 +1447,17 @@ class ArtifactSubCommand(SubCommand):
return after
@CommandProvider
class PackageFrontend(MachCommandBase):
"""Fetch and install binary artifacts from Mozilla automation."""
@Command('artifact', category='post-build',
- description='Use pre-built artifacts to build Firefox.',
- conditions=[
- conditions.is_hg, # mercurial only for now.
- ])
+ description='Use pre-built artifacts to build Firefox.')
def artifact(self):
'''Download, cache, and install pre-built binary artifacts to build Firefox.
Use |mach build| as normal to freshen your installed binary libraries:
artifact builds automatically download, cache, and install binary
artifacts from Mozilla automation, replacing whatever may be in your
object directory. Use |mach artifact last| to see what binary artifacts
were last used.
@@ -1472,36 +1469,52 @@ class PackageFrontend(MachCommandBase):
def _set_log_level(self, verbose):
self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
def _make_artifacts(self, tree=None, job=None, skip_cache=False):
self._activate_virtualenv()
self.virtualenv_manager.install_pip_package('pylru==1.0.9')
self.virtualenv_manager.install_pip_package('taskcluster==0.0.32')
+ self.virtualenv_manager.install_pip_package('mercurial==3.7.1')
self.virtualenv_manager.install_pip_package('mozregression==1.0.2')
state_dir = self._mach_context.state_dir
cache_dir = os.path.join(state_dir, 'package-frontend')
try:
os.makedirs(cache_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
import which
- if self._is_windows():
- hg = which.which('hg.exe')
- else:
- hg = which.which('hg')
+
+ here = os.path.abspath(os.path.dirname(__file__))
+ build_obj = MozbuildObject.from_environment(cwd=here)
+
+ hg = None
+ if conditions.is_hg(build_obj):
+ if self._is_windows():
+ hg = which.which('hg.exe')
+ else:
+ hg = which.which('hg')
+
+ git = None
+ if conditions.is_git(build_obj):
+ if self._is_windows():
+ git = which.which('git.exe')
+ else:
+ git = which.which('git')
+
+ python = self.virtualenv_manager.python_path
# Absolutely must come after the virtualenv is populated!
from mozbuild.artifacts import Artifacts
- artifacts = Artifacts(tree, job, log=self.log, cache_dir=cache_dir, hg=hg, skip_cache=skip_cache)
+ artifacts = Artifacts(tree, job, log=self.log, cache_dir=cache_dir, skip_cache=skip_cache, python=python, hg=hg, git=git)
return artifacts
@ArtifactSubCommand('artifact', 'install',
'Install a good pre-built artifact.')
@CommandArgument('source', metavar='SRC', nargs='?', type=str,
help='Where to fetch and install artifacts from. Can be omitted, in '
'which case the current hg repository is inspected; an hg revision; '
'a remote URL; or a local file.',