Bug 1466514 - download the uploaded wpt-manifest, r?jgraham
MozReview-Commit-ID: BP4A9S5xG7M
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -286,16 +286,25 @@ class WPTManifestUpdater(MozbuildObject)
def run_update(self, check_clean=False, rebuild=False, **kwargs):
import manifestupdate
from wptrunner import wptlogging
logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
+class WPTManifestDownloader(MozbuildObject):
+ def run_download(self, path=None, tests_root=None, force=False, **kwargs):
+ import manifestdownload
+ from wptrunner import wptlogging
+ logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
+ wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
+ manifestdownload.run(logger, wpt_dir, self.topsrcdir, force)
+
+
def create_parser_update():
from update import updatecommandline
return updatecommandline.create_parser()
def create_parser_reduce():
from wptrunner import wptcommandline
return wptcommandline.create_parser_reduce()
@@ -319,16 +328,20 @@ def create_parser_create():
p.add_argument("path", action="store", help="Path to the test file")
return p
def create_parser_manifest_update():
import manifestupdate
return manifestupdate.create_parser()
+def create_parser_manifest_download():
+ import manifestdownload
+ return manifestdownload.create_parser()
+
@CommandProvider
class MachCommands(MachCommandBase):
def setup(self):
self._activate_virtualenv()
@Command("web-platform-tests",
category="testing",
@@ -402,8 +415,17 @@ class MachCommands(MachCommandBase):
@Command("wpt-manifest-update",
category="testing",
parser=create_parser_manifest_update)
def wpt_manifest_update(self, **params):
self.setup()
wpt_manifest_updater = self._spawn(WPTManifestUpdater)
return wpt_manifest_updater.run_update(**params)
+
+
+ @Command("wpt-manifest-download",
+ category="testing",
+ parser=create_parser_manifest_download)
+ def wpt_manifest_download(self, **params):
+ self.setup()
+ wpt_manifest_downloader = self._spawn(WPTManifestDownloader)
+ return wpt_manifest_downloader.run_download(**params)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/manifestdownload.py
@@ -0,0 +1,130 @@
+from __future__ import absolute_import
+
+import argparse
+import json
+import io
+import os
+from datetime import datetime, timedelta
+import gzip
+from vcs import Mercurial
+import requests
+
+from six.moves.urllib.request import urlopen
+
+def abs_path(path):
+ return os.path.abspath(os.path.expanduser(path))
+
+
+def hg_commits(repo_root):
+ hg = Mercurial.get_func(repo_root)
+ return [item for item in hg("log", "-fl50", "--template={node}\n",
+ "testing/web-platform/tests/", "testing/web-platform/mozilla/tests").split("\n")
+ if item]
+
+
+def should_download(logger, manifest_path, rebuild_time=timedelta(days=5)):
+ # TODO: Improve logic for when to download. Maybe if x revisions behind?
+ if not os.path.exists(manifest_path):
+ return True
+ mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
+ if mtime < datetime.now() - rebuild_time:
+ return True
+ logger.info("Skipping manifest download because existing file is recent")
+ return False
+
+
+def taskcluster_url(logger, commits):
+ cset_url = ('https://hg.mozilla.org/mozilla-central/json-pushes?'
+ 'changeset={changeset}&version=2&tipsonly=1')
+
+ tc_url = ('https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.'
+ 'revision.{changeset}.source.manifest-upload')
+
+ for revision in commits:
+ req = requests.get(cset_url.format(changeset=revision),
+ headers={'Accept': 'application/json'})
+
+ req.raise_for_status()
+
+ result = req.json()
+ [cset] = result['pushes'].values()[0]['changesets']
+ resp = requests.get(tc_url.format(changeset=cset))
+
+ if req.status_code == 200:
+ return tc_url.format(changeset=cset)
+
+ logger.info("Can't find a commit-specific manifest so just using the most"
+ "recent one")
+
+ return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
+ "latest.source.manifest-upload")
+
+
+def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
+ if not force and not should_download(logger, wpt_dir):
+ return False
+
+ commits = commits_func()
+ url = url_func(logger, commits) + "/artifacts/public/"
+
+ man_url= url + "manifest.json.gz"
+ moz_man_url= url + "moz_manifest.json.gz"
+
+ return ( _download(logger, os.path.join(wpt_dir, "meta", "MANIFEST.json"), man_url) and
+ _download(logger, os.path.join(wpt_dir,"mozilla", "meta", "MANIFEST.json"), moz_man_url))
+
+
+def _download(logger, manifest_path, url):
+ if not url:
+ logger.warning("No generated manifest found")
+ return False
+
+ logger.info("Downloading manifest from %s" % url)
+ try:
+ resp = urlopen(url)
+ except Exception:
+ logger.warning("Downloading pregenerated manifest failed")
+ return False
+
+ if resp.code != 200:
+ logger.warning("Downloading pregenerated manifest failed; got"
+ "HTTP status %d" % resp.code)
+ return False
+
+ gzf = gzip.GzipFile(fileobj=io.BytesIO(resp.read()))
+
+ try:
+ decompressed = gzf.read()
+ except IOError:
+ logger.warning("Failed to decompress downloaded file")
+ return False
+
+ try:
+ with open(manifest_path, 'wb') as f:
+ f.write(decompressed)
+ except Exception as e:
+ logger.warning("Failed to write manifest at %s" % manifest_path)
+ return False
+
+ logger.info("Manifest at %s downloaded" % manifest_path)
+ return True
+
+
+def create_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "-p", "--path", type=abs_path, help="Path to manifest file.")
+ parser.add_argument(
+ "--force", action="store_true",
+ help="Always download, even if the existing manifest is recent")
+ return parser
+
+
+def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
+ return download_manifest(logger, wpt_dir, lambda: hg_commits(repo_root),
+ taskcluster_url, force)
+
+
+def run(logger, wpt_dir, repo_root, force=False):
+ success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
+ return 0 if success else 1
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/vcs.py
@@ -0,0 +1,20 @@
+import os
+import subprocess
+
+class Mercurial(object):
+
+ def __init__(self, repo_root):
+ self.root = os.path.abspath(repo_root)
+ self.hg = Mercurial.get_func(repo_root)
+
+
+ @staticmethod
+ def get_func(repo_path):
+ def hg(cmd, *args):
+ full_cmd = ["hg", cmd] + list(args)
+ try:
+ return subprocess.check_output(full_cmd, cwd=repo_path, stderr=subprocess.STDOUT)
+ except Exception as e:
+ raise(e)
+ # TODO: Test on Windows.
+ return hg