Bug 1234500 - Allow mach talos-test to download tp5 pageset; r?jmaher
MozReview-Commit-ID: Jj5H3G2hLg0
--- a/.hgignore
+++ b/.hgignore
@@ -110,25 +110,27 @@ GPATH
# Ignore tox generated dir
.tox/
# Ignore node_modules
^tools/lint/eslint/node_modules/
# Ignore talos virtualenv and tp5n files.
# The tp5n set is supposed to be decompressed at
-# testing/talos/talos/page_load_test/tp5n in order to run tests like tps
+# testing/talos/talos/tests/tp5n in order to run tests like tps
# locally. Similarly, running talos requires a Python package virtual
# environment. Both the virtual environment and tp5n files end up littering
# the status command, so we ignore them.
^testing/talos/.Python
^testing/talos/bin/
^testing/talos/include/
^testing/talos/lib/
^testing/talos/talos/tests/tp5n.zip
+^testing/talos/talos/tests/tp5n.tar.gz
^testing/talos/talos/tests/tp5n
^testing/talos/talos/tests/devtools/damp.manifest.develop
+^talos-venv
# Ignore files created when running a reftest.
^lextab.py$
# tup database
^\.tup
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -20,16 +20,17 @@ from mozharness.base.errors import Pytho
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL
from mozharness.base.log import INFO, WARNING
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
+from mozharness.mozilla.tooltool import TooltoolMixin
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
)
TalosErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
@@ -81,17 +82,17 @@ class TalosOutputParser(OutputParser):
harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
if harness_retry_re.search(line):
self.critical(' %s' % line)
self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
return # skip base parse_single_line
super(TalosOutputParser, self).parse_single_line(line)
-class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
+class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin):
"""
install and run Talos tests:
https://wiki.mozilla.org/Buildbot/Talos
"""
config_options = [
[["--use-talos-json"],
{"action": "store_true",
"dest": "use_talos_json",
@@ -159,19 +160,19 @@ class Talos(TestingMixin, MercurialScrip
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
self.run_local = self.config.get('run_local')
self.installer_url = self.config.get("installer_url")
self.talos_json_url = self.config.get("talos_json_url")
self.talos_json = self.config.get("talos_json")
self.talos_json_config = self.config.get("talos_json_config")
self.tests = None
- self.pagesets_url = None
self.sps_profile = self.config.get('sps_profile')
self.sps_profile_interval = self.config.get('sps_profile_interval')
+ self.pagesets_name = None
# We accept some configuration options from the try commit message in the format mozharness: <options>
# Example try commit message:
# mozharness: --spsProfile try: <stuff>
def query_sps_profile_options(self):
sps_results = []
if self.buildbot_config:
# this is inside automation
@@ -218,25 +219,52 @@ class Talos(TestingMixin, MercurialScrip
if self.talos_json_config:
return self.talos_json_config
if not self.talos_json:
self.talos_json = os.path.join(self.talos_path, 'talos.json')
self.talos_json_config = parse_config_file(self.talos_json)
self.info(pprint.pformat(self.talos_json_config))
return self.talos_json_config
- def query_pagesets_url(self):
+ def query_pagesets_name(self):
"""Certain suites require external pagesets to be downloaded and
extracted.
"""
- if self.pagesets_url:
- return self.pagesets_url
- if self.query_talos_json_config() and 'suite' in self.config:
- self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
- return self.pagesets_url
+ if self.pagesets_name:
+ return self.pagesets_name
+ if self.query_talos_json_config() and self.suite is not None:
+ self.pagesets_name = self.talos_json_config['suites'][self.suite].get('pagesets_name')
+ return self.pagesets_name
+
+ def get_suite_from_test(self):
+ """ Retrieve the talos suite name from a given talos test name."""
+ # running locally, single test name provided instead of suite; go through tests and find suite name
+ suite_name = None
+ if self.query_talos_json_config():
+ if '-a' in self.config['talos_extra_options']:
+ test_name_index = self.config['talos_extra_options'].index('-a') + 1
+ if '--activeTests' in self.config['talos_extra_options']:
+ test_name_index = self.config['talos_extra_options'].index('--activeTests') + 1
+ if test_name_index < len(self.config['talos_extra_options']):
+ test_name = self.config['talos_extra_options'][test_name_index]
+ for talos_suite in self.talos_json_config['suites']:
+ if test_name in self.talos_json_config['suites'][talos_suite].get('tests'):
+ suite_name = talos_suite
+ if not suite_name:
+ # no suite found to contain the specified test, error out
+ self.fatal("Test name is missing or invalid")
+ else:
+ self.fatal("Talos json config not found, cannot verify suite")
+ return suite_name
+
+ def validate_suite(self):
+ """ Ensure suite name is a valid talos suite. """
+ if self.query_talos_json_config() and self.suite is not None:
+ if not self.suite in self.talos_json_config.get('suites'):
+ self.fatal("Suite '%s' is not valid (not found in talos json config)" % self.suite)
def talos_options(self, args=None, **kw):
"""return options to talos"""
# binary path
binary_path = self.binary_path or self.config.get('binary_path')
if not binary_path:
self.fatal("Talos requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.")
@@ -269,31 +297,51 @@ class Talos(TestingMixin, MercurialScrip
if args is not None:
options += args
if 'talos_extra_options' in self.config:
options += self.config['talos_extra_options']
return options
def populate_webroot(self):
"""Populate the production test slaves' webroots"""
- c = self.config
-
self.talos_path = os.path.join(
self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
)
- if c.get('run_local'):
- self.talos_path = os.path.dirname(self.talos_json)
-
- src_talos_webdir = os.path.join(self.talos_path, 'talos')
- if self.query_pagesets_url():
- self.info('Downloading pageset...')
- dirs = self.query_abs_dirs()
- src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
- archive = self.download_file(self.pagesets_url, parent_dir=dirs['abs_work_dir'])
+ # need to determine if talos pageset is required to be downloaded
+ if self.config.get('run_local'):
+ # talos initiated locally, get and verify test/suite from cmd line
+ self.talos_path = os.path.dirname(self.talos_json)
+ if '-a' in self.config['talos_extra_options'] or '--activeTests' in self.config['talos_extra_options']:
+ # test name (-a or --activeTests) specified, find out what suite it is a part of
+ self.suite = self.get_suite_from_test()
+ elif '--suite' in self.config['talos_extra_options']:
+ # --suite specified, get suite from cmd line and ensure is valid
+ suite_name_index = self.config['talos_extra_options'].index('--suite') + 1
+ if suite_name_index < len(self.config['talos_extra_options']):
+ self.suite = self.config['talos_extra_options'][suite_name_index]
+ self.validate_suite()
+ else:
+ self.fatal("Suite name not provided")
+ else:
+ # talos initiated in production via mozharness
+ self.suite = self.config['suite']
+
+ # now that have the suite name, check if pageset is required, if so download it
+ if self.query_pagesets_name():
+ self.info("Downloading pageset with tooltool...")
+ self.src_talos_webdir = os.path.join(self.talos_path, 'talos')
+ src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests')
+ manifest_file = os.path.join(self.talos_path, 'tp5n-pageset.manifest')
+ self.tooltool_fetch(
+ manifest_file,
+ output_dir=src_talos_pageset,
+ cache=self.config.get('tooltool_cache')
+ )
+ archive = os.path.join(src_talos_pageset, self.pagesets_name)
unzip = self.query_exe('unzip')
unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
self.run_command(unzip_cmd, halt_on_failure=True)
# Action methods. {{{1
# clobber defined in BaseScript
# read_buildbot_config defined in BuildbotMixin
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -61,16 +61,17 @@ class TalosRunner(MozbuildObject):
'virtualenv': [self.python_interp, self.virtualenv_script]
},
'title': socket.gethostname(),
'default_actions': [
'populate-webroot',
'create-virtualenv',
'run-tests',
],
+ 'download_tooltool': True,
'talos_extra_options': ['--develop'] + self.talos_args,
}
def make_args(self):
self.args = {
'config': {},
'initial_config_file': self.config_file_path,
}
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -40,30 +40,30 @@
"talos_options": ["--disable-e10s"]
},
"other-e10s_l64": {
"tests": ["a11yr", "ts_paint", "tpaint", "sessionrestore", "sessionrestore_no_auto_restore", "tabpaint"]
},
"g1": {
"tests": ["tp5o_scroll", "glterrain"],
"talos_options": ["--disable-e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip"
},
"g1-e10s": {
"tests": ["tp5o_scroll", "glterrain"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip"
},
"g2": {
"tests": ["damp", "tps"],
"talos_options": ["--disable-e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip"
},
"g2-e10s": {
"tests": ["damp", "tps"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip"
},
"g3": {
"tests": ["dromaeo_dom"],
"talos_options": ["--disable-e10s"]
},
"g3-e10s": {
"tests": ["dromaeo_dom"]
},
@@ -78,35 +78,35 @@
"tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"],
"talos_options": ["--disable-e10s"]
},
"svgr-e10s": {
"tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"]
},
"tp5o": {
"tests": ["tp5o"],
- "talos_options": ["--disable-e10s"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip",
+ "talos_options": ["--disable-e10s"]
},
"tp5o-e10s": {
"tests": ["tp5o"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+ "pagesets_name": "tp5n.zip"
},
"xperf": {
"tests": ["tp5n"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+ "pagesets_name": "tp5n.zip",
"talos_options": [
"--disable-e10s",
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
},
"xperf-e10s": {
"tests": ["tp5n"],
- "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+ "pagesets_name": "tp5n.zip",
"talos_options": [
"--xperf_path",
"\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
]
}
},
"mobile-suites": {
"remote-tsvgx": {
new file mode 100644
--- /dev/null
+++ b/testing/talos/tp5n-pageset.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "tp5n.zip",
+ "size": 81753769,
+ "digest": "7e74bc532d220fa2484f84bd7c2659da7d2ae3aa0bc225ba63e3db70dc0c0697503427209098afa85e235397c4ec58cd488cab7b3435e8079583d3994fff8326",
+ "algorithm": "sha512",
+ "unpack": false
+ }
+]
\ No newline at end of file