Bug 1234500 - allow for ./mach talos-test to download tp5 pageset and use it properly. r=jmaher
--- a/.hgignore
+++ b/.hgignore
@@ -110,17 +110,11 @@ GPATH
# Ignore tox generated dir
.tox/
# Ignore node_modules from eslint-plugin-mozilla
^testing/eslint-plugin-mozilla/node_modules/
# Ignore talos virtualenv and tp5n files.
# The tp5n set is supposed to be decompressed at
-# testing/talos/talos/page_load_test/tp5n in order to run tests like tps
-# locally. Similarly, running talos requires a Python package virtual
-# environment. Both the virtual environment and tp5n files end up littering
-# the status command, so we ignore them.
-^testing/talos/.Python
-^testing/talos/bin/
-^testing/talos/include/
-^testing/talos/lib/
-^testing/talos/talos/page_load_test/tp5n/
+# testing/talos/talos/tests/tp5n in order to run tests like tps locally.
+^testing/talos/talos/tests/tp5n.tar.gz
+^testing/talos/talos/tests/tp5n/
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -19,16 +19,17 @@ from mozharness.base.errors import Pytho
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, FATAL
from mozharness.base.log import INFO, WARNING
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options, INSTALLER_SUFFIXES
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
+from mozharness.mozilla.tooltool import TooltoolMixin
TalosErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
{'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
{'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
{'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
{'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
{'substr': r'''talosError''', 'level': CRITICAL},
@@ -75,17 +76,17 @@ class TalosOutputParser(OutputParser):
harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
if harness_retry_re.search(line):
self.critical(' %s' % line)
self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
return # skip base parse_single_line
super(TalosOutputParser, self).parse_single_line(line)
-class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
+class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin):
"""
install and run Talos tests:
https://wiki.mozilla.org/Buildbot/Talos
"""
config_options = [
[["--use-talos-json"],
{"action": "store_true",
"dest": "use_talos_json",
@@ -153,17 +154,16 @@ class Talos(TestingMixin, MercurialScrip
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
self.run_local = self.config.get('run_local')
self.installer_url = self.config.get("installer_url")
self.talos_json_url = self.config.get("talos_json_url")
self.talos_json = self.config.get("talos_json")
self.talos_json_config = self.config.get("talos_json_config")
self.tests = None
- self.pagesets_url = None
self.sps_profile = self.config.get('sps_profile')
self.sps_profile_interval = self.config.get('sps_profile_interval')
# We accept some configuration options from the try commit message in the format mozharness: <options>
# Example try commit message:
# mozharness: --spsProfile try: <stuff>
def query_sps_profile_options(self):
sps_results = []
@@ -205,26 +205,16 @@ class Talos(TestingMixin, MercurialScrip
if self.talos_json_config:
return self.talos_json_config
if not self.talos_json:
self.talos_json = os.path.join(self.talos_path, 'talos.json')
self.talos_json_config = parse_config_file(self.talos_json)
self.info(pprint.pformat(self.talos_json_config))
return self.talos_json_config
- def query_pagesets_url(self):
- """Certain suites require external pagesets to be downloaded and
- extracted.
- """
- if self.pagesets_url:
- return self.pagesets_url
- if self.query_talos_json_config() and 'suite' in self.config:
- self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
- return self.pagesets_url
-
def talos_options(self, args=None, **kw):
"""return options to talos"""
# binary path
binary_path = self.binary_path or self.config.get('binary_path')
if not binary_path:
self.fatal("Talos requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.")
# talos options
@@ -264,22 +254,22 @@ class Talos(TestingMixin, MercurialScrip
c = self.config
self.talos_path = os.path.join(
self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
)
if c.get('run_local'):
self.talos_path = os.path.dirname(self.talos_json)
- src_talos_webdir = os.path.join(self.talos_path, 'talos')
-
- if self.query_pagesets_url():
- self.info("Downloading pageset...")
- src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
- self.download_unzip(self.pagesets_url, src_talos_pageset)
+ self.info("Downloading pageset with tooltool...")
+ self.tooltool_fetch(
+ os.path.join(self.talos_path, 'pageset-tooltool.manifest'),
+ output_dir=os.path.join(self.talos_path, 'talos', 'tests'),
+ cache=c.get('tooltool_cache')
+ )
# Action methods. {{{1
# clobber defined in BaseScript
# read_buildbot_config defined in BuildbotMixin
def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
return super(Talos, self).download_and_extract(
suite_categories=['common', 'talos']
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -61,16 +61,17 @@ class TalosRunner(MozbuildObject):
'virtualenv': [self.python_interp, self.virtualenv_script]
},
'title': socket.gethostname(),
'default_actions': [
'populate-webroot',
'create-virtualenv',
'run-tests',
],
+ 'download_tooltool': True,
'talos_extra_options': ['--develop'] + self.talos_args,
}
def make_args(self):
self.args = {
'config': {},
'initial_config_file': self.config_file_path,
}
new file mode 100644
--- /dev/null
+++ b/testing/talos/pageset-tooltool.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "tp5n.tar.gz",
+ "size": 75879655,
+ "digest": "8cee387b2e38421d4b2d457c5762cacd7a5e4deb82c689fed2665038b0daf8e00fe7b0f76173a5d69de62a6e9c6d32652c710e30e1293f33d76c4ae221e7ce30",
+ "algorithm": "sha512",
+ "unpack": true
+ }
+]
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -38,66 +38,58 @@
"other_l64": {
"tests": ["a11yr", "ts_paint", "tpaint", "sessionrestore", "sessionrestore_no_auto_restore"]
},
"other-e10s_l64": {
"tests": ["a11yr", "ts_paint", "tpaint", "sessionrestore", "sessionrestore_no_auto_restore"],
"talos_options": ["--e10s"]
},
"g1": {
- "tests": ["tp5o_scroll", "glterrain"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "tests": ["tp5o_scroll", "glterrain"]
},
"g1-e10s": {
"tests": ["tp5o_scroll", "glterrain"],
- "talos_options": ["--e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "talos_options": ["--e10s"]
},
"g2": {
- "tests": ["damp", "tps"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "tests": ["damp", "tps"]
},
"g2-e10s": {
"tests": ["damp", "tps"],
- "talos_options": ["--e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "talos_options": ["--e10s"]
},
"g3": {
"tests": ["dromaeo_dom"]
},
"g3-e10s": {
"tests": ["dromaeo_dom"]
},
"svgr": {
"tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"]
},
"svgr-e10s": {
"tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"],
"talos_options": ["--e10s"]
},
"tp5o": {
- "tests": ["tp5o"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "tests": ["tp5o"]
},
"tp5o-e10s": {
"tests": ["tp5o"],
- "talos_options": ["--e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "talos_options": ["--e10s"]
},
"xperf": {
"tests": ["tp5n"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
"talos_options": [
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
},
"xperf-e10s": {
"tests": ["tp5n"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
"talos_options": [
"--e10s",
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
}
},
"mobile-suites": {