Bug 1234913 - Part 2: Support git in |mach artifact install|. r?chmanchester draft
authorNick Alexander <nalexander@mozilla.com>
Wed, 24 Feb 2016 23:20:42 -0800
changeset 334425 4783345d9b2b65028757914af71ee48b767763f2
parent 334424 847836fdbcaf930894f05c7c555a1ed319c130d7
child 334426 6f2a6fb3e295d0178011a769e8e999f7f9fd1b61
push id11551
push usernalexander@mozilla.com
push dateThu, 25 Feb 2016 07:47:43 +0000
reviewerschmanchester
bugs1234913
milestone47.0a1
Bug 1234913 - Part 2: Support git in |mach artifact install|. r?chmanchester MozReview-Commit-ID: LL6kO8QS5p9
python/mozbuild/mozbuild/artifacts.py
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -675,21 +675,27 @@ class ArtifactCache(CacheManager):
         self.log(logging.INFO, 'artifact',
             {'filename': result + PROCESSED_SUFFIX},
             'Last installed binaries from local processed file {filename}')
 
 
 class Artifacts(object):
     '''Maintain state to efficiently fetch build artifacts from a Firefox tree.'''
 
-    def __init__(self, tree, job=None, log=None, cache_dir='.', hg='hg', skip_cache=False):
+    def __init__(self, tree, job=None, log=None, cache_dir='.', python=None, hg=None, git=None, skip_cache=False):
         self._tree = tree
         self._job = job or self._guess_artifact_job()
         self._log = log
+        if not python:
+            raise ValueError("Must provide path to Python")
+        if (hg and git) or (not hg and not git):
+            raise ValueError("Must provide path to exactly one of hg and git")
+        self._python = python
         self._hg = hg
+        self._git = git
         self._cache_dir = cache_dir
         self._skip_cache = skip_cache
 
         try:
             self._artifact_job = get_job_details(self._job, log=self._log)
         except KeyError:
             self.log(logging.INFO, 'artifact',
                 {'job': self._job},
@@ -770,16 +776,58 @@ class Artifacts(object):
             yield rev_info[0], tuple(rev_info[1:])
 
         if not count:
             raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n\n'
                             'Try running |hg pushlogsync|;\n'
                             'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format(
                                 num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT))
 
+    def _find_git_pushheads(self, rev):
+        """Return an iterator of (hg_hash, {tree-set}) associating hg revision
+        hashes that might be pushheads with the trees they are known
+        to be in.
+
+        More recent hashes should come earlier in the list.  It's okay
+        for tree-set to be the empty set {}; in that case, we'll query
+        the TaskCluster Index to determine the tree-set.
+        """
+
+        import which
+        cinnabar = which.which('git-cinnabar')
+
+        # First commit is HEAD, next is HEAD~1, etc.
+        rev_list = subprocess.check_output([
+            self._git, 'rev-list', '--ancestry-path',
+            'HEAD~{num}..HEAD'.format(num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT),
+        ])
+
+        hg_hash_list = subprocess.check_output([
+            self._python, cinnabar, 'git2hg',
+        ] + rev_list.splitlines())
+
+        zeroes = "0" * 40
+        self._index = taskcluster.Index()
+
+        # We don't have pushlog data, so we never know there is a push
+        # (resulting in any kind of job) corresponding to any trees.
+        trees = tuple()
+        count = 0
+        for hg_hash in hg_hash_list.splitlines():
+            hg_hash = hg_hash.strip()
+            if not hg_hash or hg_hash == zeroes:
+                continue
+            count += 1
+            yield (hg_hash, trees)
+
+        if not count:
+            raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n\n'
+                            'See https://developer.mozilla.org/en-US/docs/Artifact_builds'.format(
+                                num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT))
+
     def find_pushhead_artifacts(self, task_cache, tree_cache, job, pushhead, trees):
         known_trees = set(tree_cache.artifact_trees(pushhead, trees))
         if not known_trees:
             return None
         if not trees:
             # Accept artifacts from any tree where they are available.
             trees = list(known_trees)
             trees.sort()
@@ -886,16 +934,20 @@ class Artifacts(object):
                  {'count': count},
                  'Tried {count} pushheads, no built artifacts found.')
         return 1
 
     def install_from_hg_recent(self, distdir):
         hg_pushheads = self._find_hg_pushheads()
         return self._install_from_hg_pushheads(hg_pushheads, distdir)
 
+    def install_from_git_recent(self, distdir):
+        hg_pushheads = self._find_git_pushheads('HEAD')
+        return self._install_from_hg_pushheads(hg_pushheads, distdir)
+
     def install_from_hg_revset(self, revset, distdir):
         revision = subprocess.check_output([self._hg, 'log', '--template', '{node}\n',
                                             '-r', revset]).strip()
         if len(revision.split('\n')) != 1:
             raise ValueError('hg revision specification must resolve to exactly one commit')
         hg_pushheads = [(revision, tuple())]
         self.log(logging.INFO, 'artifact',
                  {'revset': revset,
@@ -911,18 +963,24 @@ class Artifacts(object):
             return self.install_from_file(source, distdir)
         elif source and urlparse.urlparse(source).scheme:
             return self.install_from_url(source, distdir)
         else:
             if source is None and 'MOZ_ARTIFACT_REVISION' in os.environ:
                 source = os.environ['MOZ_ARTIFACT_REVISION']
 
             if source:
+                if self._git:
+                    # TODO: Handle MOZ_ARTIFACT_{HG,GIT}_REVISION.
+                    raise ValueError("MOZ_ARTIFACT_REVISION is not yet supported with git")
                 return self.install_from_hg_revset(source, distdir)
 
+            if self._git:
+                return self.install_from_git_recent(distdir)
+
             return self.install_from_hg_recent(distdir)
 
 
     def print_last(self):
         self.log(logging.INFO, 'artifact',
             {},
             'Printing last used artifact details.')
         self._tree_cache.print_last()
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1447,20 +1447,17 @@ class ArtifactSubCommand(SubCommand):
         return after
 
 
 @CommandProvider
 class PackageFrontend(MachCommandBase):
     """Fetch and install binary artifacts from Mozilla automation."""
 
     @Command('artifact', category='post-build',
-        description='Use pre-built artifacts to build Firefox.',
-        conditions=[
-            conditions.is_hg,  # mercurial only for now.
-        ])
+        description='Use pre-built artifacts to build Firefox.')
     def artifact(self):
         '''Download, cache, and install pre-built binary artifacts to build Firefox.
 
         Use |mach build| as normal to freshen your installed binary libraries:
         artifact builds automatically download, cache, and install binary
         artifacts from Mozilla automation, replacing whatever may be in your
         object directory.  Use |mach artifact last| to see what binary artifacts
         were last used.
@@ -1472,36 +1469,52 @@ class PackageFrontend(MachCommandBase):
 
     def _set_log_level(self, verbose):
         self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
 
     def _make_artifacts(self, tree=None, job=None, skip_cache=False):
         self._activate_virtualenv()
         self.virtualenv_manager.install_pip_package('pylru==1.0.9')
         self.virtualenv_manager.install_pip_package('taskcluster==0.0.32')
+        self.virtualenv_manager.install_pip_package('mercurial==3.7.1')
         self.virtualenv_manager.install_pip_package('mozregression==1.0.2')
 
         state_dir = self._mach_context.state_dir
         cache_dir = os.path.join(state_dir, 'package-frontend')
 
         try:
             os.makedirs(cache_dir)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         import which
-        if self._is_windows():
-          hg = which.which('hg.exe')
-        else:
-          hg = which.which('hg')
+
+        here = os.path.abspath(os.path.dirname(__file__))
+        build_obj = MozbuildObject.from_environment(cwd=here)
+
+        hg = None
+        if conditions.is_hg(build_obj):
+            if self._is_windows():
+                hg = which.which('hg.exe')
+            else:
+                hg = which.which('hg')
+
+        git = None
+        if conditions.is_git(build_obj):
+            if self._is_windows():
+                git = which.which('git.exe')
+            else:
+                git = which.which('git')
+
+        python = self.virtualenv_manager.python_path
 
         # Absolutely must come after the virtualenv is populated!
         from mozbuild.artifacts import Artifacts
-        artifacts = Artifacts(tree, job, log=self.log, cache_dir=cache_dir, hg=hg, skip_cache=skip_cache)
+        artifacts = Artifacts(tree, job, log=self.log, cache_dir=cache_dir, skip_cache=skip_cache, python=python, hg=hg, git=git)
         return artifacts
 
     @ArtifactSubCommand('artifact', 'install',
         'Install a good pre-built artifact.')
     @CommandArgument('source', metavar='SRC', nargs='?', type=str,
         help='Where to fetch and install artifacts from.  Can be omitted, in '
             'which case the current hg repository is inspected; an hg revision; '
             'a remote URL; or a local file.',