Bug 1468869 - Add git-cinnabar support for wpt-manifest download, r?ato draft
authorAhilya Sinha <asinha@mozilla.com>
Thu, 14 Jun 2018 15:51:26 -0700
changeset 809610 61dfc8f6008e944ecc4be65cf15de69644cc6521
parent 806530 8ab6afabc78cd909ff90ba74c1eab098985f83ef
push id113725
push userbmo:asinha@mozilla.com
push dateFri, 22 Jun 2018 15:10:48 +0000
reviewersato
bugs1468869
milestone62.0a1
Bug 1468869 - Add git-cinnabar support for wpt-manifest download, r?ato MozReview-Commit-ID: 7Nl4r7ty7Ji
testing/web-platform/manifestdownload.py
testing/web-platform/vcs.py
--- a/testing/web-platform/manifestdownload.py
+++ b/testing/web-platform/manifestdownload.py
@@ -1,28 +1,46 @@
 from __future__ import absolute_import
 
 import argparse
 import json
 import os
 from datetime import datetime, timedelta
 import tarfile
-from vcs import Mercurial
+import vcs
 import requests
 from cStringIO import StringIO
 
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
 def hg_commits(repo_root):
-    hg = Mercurial.get_func(repo_root)
-    return [item for item in hg("log", "-fl50", "--template={node}\n",
-            "testing/web-platform/tests/", "testing/web-platform/mozilla/tests").split("\n")
-            if item]
+    hg = vcs.Mercurial.get_func(repo_root)
+    for item in hg("log", "-fl50", "--template={node}\n", "testing/web-platform/tests",
+                   "testing/web-platform/mozilla/tests").splitlines():
+        yield item
+
+
+def git_commits(repo_root):
+    git = vcs.Git.get_func(repo_root)
+    for item in git("log", "--format=%H", "-n50", "testing/web-platform/tests",
+                    "testing/web-platform/mozilla/tests").splitlines():
+        yield git("cinnabar", "git2hg", item)
+
+
+def get_commits(logger, repo_root):
+    if vcs.Mercurial.is_hg_repo(repo_root):
+        return hg_commits(repo_root)
+
+    elif vcs.Git.is_git_repo(repo_root):
+        return git_commits(repo_root)
+
+    logger.warning("No VCS found")
+    return False
 
 
 def should_download(logger, manifest_path, rebuild_time=timedelta(days=5)):
     # TODO: Improve logic for when to download. Maybe if x revisions behind?
     if not os.path.exists(manifest_path):
         return True
     mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
     if mtime < datetime.now() - rebuild_time:
@@ -54,20 +72,22 @@ def taskcluster_url(logger, commits):
     logger.info("Can't find a commit-specific manifest so just using the most"
                 "recent one")
 
     return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
             "latest.source.manifest-upload")
 
 
 def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
-    if not force and not should_download(logger, wpt_dir):
+    if not force and not should_download(logger, os.path.join(wpt_dir, "meta", "MANIFEST.json")):
         return False
 
     commits = commits_func()
+    if not commits:
+        return False
     url = url_func(logger, commits) + "/artifacts/public/manifests.tar.gz"
 
     if not url:
         logger.warning("No generated manifest found")
         return False
 
     logger.info("Downloading manifest from %s" % url)
     try:
@@ -101,15 +121,15 @@ def create_parser():
         "-p", "--path", type=abs_path, help="Path to manifest file.")
     parser.add_argument(
         "--force", action="store_true",
         help="Always download, even if the existing manifest is recent")
     return parser
 
 
 def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
-    return download_manifest(logger, wpt_dir, lambda: hg_commits(repo_root),
+    return download_manifest(logger, wpt_dir, lambda: get_commits(logger, repo_root),
                              taskcluster_url, force)
 
 
 def run(logger, wpt_dir, repo_root, force=False):
     success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
     return 0 if success else 1
--- a/testing/web-platform/vcs.py
+++ b/testing/web-platform/vcs.py
@@ -1,20 +1,51 @@
 import os
 import subprocess
 
 class Mercurial(object):
-
     def __init__(self, repo_root):
         self.root = os.path.abspath(repo_root)
         self.hg = Mercurial.get_func(repo_root)
 
-
     @staticmethod
-    def get_func(repo_path):
+    def get_func(repo_root):
         def hg(cmd, *args):
             full_cmd = ["hg", cmd] + list(args)
-            try:
-                return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
-            except Exception as e:
-                raise(e)
+            return subprocess.check_output(full_cmd, cwd=repo_root)
             # TODO: Test on Windows.
         return hg
+
+    @staticmethod
+    def is_hg_repo(repo_root):
+        try:
+            with open(os.devnull, 'w') as devnull:
+                subprocess.check_call(["hg", "root"], cwd=repo_root, stdout=devnull,
+                                        stderr=devnull)
+        except subprocess.CalledProcessError:
+            return False
+        # TODO: Test on windows
+        return True
+
+
+class Git(object):
+    def __init__(self, repo_root, url_base):
+        self.root = os.path.abspath(repo_root)
+        self.git = Git.get_func(repo_root)
+
+    @staticmethod
+    def get_func(repo_root):
+        def git(cmd, *args):
+            full_cmd = ["git", cmd] + list(args)
+            return subprocess.check_output(full_cmd, cwd=repo_root)
+            # TODO: Test on Windows.
+        return git
+
+    @staticmethod
+    def is_git_repo(repo_root):
+        try:
+            with open(os.devnull, 'w') as devnull:
+                subprocess.check_call(["git", "rev-parse", "--show-cdup"], cwd=repo_root,
+                                        stdout=devnull, stderr=devnull)
+        except subprocess.CalledProcessError:
+            return False
+        # TODO: Test on windows
+        return True