Bug 1372324 - Enable talos tests on linux64-ccov. r?jmaher
This patch enables talos test suites to run on VM (taskcluster) and also enables these test suites to run with GCOV code coverage instrumentation on the linux64-ccov build.
MozReview-Commit-ID: 7p59zvra1ge
--- a/browser/config/mozconfigs/linux64/code-coverage
+++ b/browser/config/mozconfigs/linux64/code-coverage
@@ -3,13 +3,14 @@
TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir}
ac_add_options --disable-install-strip
ac_add_options --disable-jemalloc
ac_add_options --disable-crashreporter
ac_add_options --disable-elf-hack
ac_add_options --enable-debug
ac_add_options --disable-sandbox
+ac_add_options --disable-profiling
MOZ_CODE_COVERAGE=1
export CFLAGS="--coverage"
export CXXFLAGS="--coverage"
export LDFLAGS="--coverage -L$TOOLTOOL_DIR/gtk3/usr/local/lib"
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -118,16 +118,17 @@ linux64-qr/debug:
- qr-tests
linux64-ccov/opt:
build-platform: linux64-ccov/opt
test-sets:
- common-tests
- web-platform-tests
- awsy
+ - talos
- headless
linux64-jsdcov/opt:
build-platform: linux64-jsdcov/opt
test-sets:
- jsdcov-code-coverage-tests
##
# Windows platforms (matching /windows.*/)
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -504,16 +504,26 @@ def handle_keyed_by(config, tests):
def enable_code_coverage(config, tests):
"""Enable code coverage for the linux64-ccov/opt & linux64-jsdcov/opt build-platforms"""
for test in tests:
if test['build-platform'] == 'linux64-ccov/opt':
test['mozharness'].setdefault('extra-options', []).append('--code-coverage')
test['when'] = {}
test['instance-size'] = 'xlarge'
test['run-on-projects'] = ['mozilla-central']
+
+ if test['test-name'].startswith('talos'):
+ test['max-run-time'] = 7200
+ test['mozharness']['config'] = ['talos/linux64_config_taskcluster.py']
+ test['mozharness']['extra-options'].append('--add-option')
+ test['mozharness']['extra-options'].append('--cycles,1')
+ test['mozharness']['extra-options'].append('--add-option')
+ test['mozharness']['extra-options'].append('--tppagecycles,1')
+ test['mozharness']['extra-options'].append('--add-option')
+ test['mozharness']['extra-options'].append('--no-upload-results')
elif test['build-platform'] == 'linux64-jsdcov/opt':
test['run-on-projects'] = ['mozilla-central']
yield test
@transforms.add
def handle_run_on_projects(config, tests):
"""Handle translating `built-projects` appropriately"""
@@ -699,17 +709,17 @@ def set_worker_type(config, tests):
# note that some portion of these will be allocated to BBB below
test['worker-type'] = MACOSX_WORKER_TYPES['macosx64']
elif test_platform.startswith('win'):
if test.get('suite', '') == 'talos':
test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
else:
test['worker-type'] = WINDOWS_WORKER_TYPES[test_platform.split('/')[0]]
elif test_platform.startswith('linux') or test_platform.startswith('android'):
- if test.get('suite', '') == 'talos':
+ if test.get('suite', '') == 'talos' and test['build-platform'] != 'linux64-ccov/opt':
if config.config['args'].taskcluster_worker:
test['worker-type'] = 'releng-hardware/gecko-t-linux-talos'
else:
test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
else:
test['worker-type'] = LINUX_WORKER_TYPES[test['instance-size']]
else:
raise Exception("unknown test_platform {}".format(test_platform))
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/talos/linux64_config_taskcluster.py
@@ -0,0 +1,54 @@
+import os
+import platform
+import sys
+
+PYTHON = sys.executable
+VENV_PATH = '%s/build/venv' % os.getcwd()
+
+if platform.architecture()[0] == '64bit':
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+ TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+ MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+
+exes = {
+ 'tooltool.py': ["/builds/tooltool.py"],
+ 'python': PYTHON,
+ 'virtualenv': [PYTHON, '/usr/local/lib/python2.7/dist-packages/virtualenv.py'],
+}
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+ "log_name": "talos",
+ "buildbot_json_path": "buildprops.json",
+ "download_tooltool": True,
+ "installer_path": INSTALLER_PATH,
+ "virtualenv_path": VENV_PATH,
+ "find_links": [
+ "http://pypi.pvt.build.mozilla.org/pub",
+ "http://pypi.pub.build.mozilla.org/pub",
+ ],
+ "pip_index": False,
+ "exes": exes,
+ "title": os.uname()[1].lower().split('.')[0],
+ "default_actions": [
+ "clobber",
+ "read-buildbot-config",
+ "download-and-extract",
+ "populate-webroot",
+ "create-virtualenv",
+ "install",
+ "setup-mitmproxy",
+ "run-tests",
+ ],
+ "default_blob_upload_servers": [
+ "https://blobupload.elasticbeanstalk.com",
+ ],
+ "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+ "download_minidump_stackwalk": True,
+ "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+ "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+ "tooltool_cache": "/home/worker/tooltool-cache",
+}
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -24,16 +24,21 @@ from mozharness.base.log import INFO, WA
from mozharness.base.python import Python3Virtualenv
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options
+)
+
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, 'external_tools')
TalosErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
{'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
{'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
@@ -84,17 +89,17 @@ class TalosOutputParser(OutputParser):
if harness_retry_re.search(line):
self.critical(' %s' % line)
self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
return # skip base parse_single_line
super(TalosOutputParser, self).parse_single_line(line)
class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin,
- Python3Virtualenv):
+ Python3Virtualenv, CodeCoverageMixin):
"""
install and run Talos tests:
https://wiki.mozilla.org/Buildbot/Talos
"""
config_options = [
[["--use-talos-json"],
{"action": "store_true",
"dest": "use_talos_json",
@@ -132,17 +137,18 @@ class Talos(TestingMixin, MercurialScrip
"help": "Whether or not to profile the test run and save the profile results"
}],
[["--geckoProfileInterval"], {
"dest": "gecko_profile_interval",
"type": "int",
"default": 0,
"help": "The interval between samples taken by the profiler (milliseconds)"
}],
- ] + testing_config_options + copy.deepcopy(blobupload_config_options)
+ ] + testing_config_options + copy.deepcopy(blobupload_config_options) \
+ + copy.deepcopy(code_coverage_config_options)
def __init__(self, **kwargs):
kwargs.setdefault('config_options', self.config_options)
kwargs.setdefault('all_actions', ['clobber',
'read-buildbot-config',
'download-and-extract',
'populate-webroot',
'create-virtualenv',
@@ -554,17 +560,17 @@ class Talos(TestingMixin, MercurialScrip
if self.return_code == 1:
log_level = WARNING
tbpl_level = TBPL_WARNING
if self.return_code == 4:
log_level = WARNING
tbpl_level = TBPL_RETRY
parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level)
- else:
+ elif '--no-upload-results' not in options:
if not self.gecko_profile:
self._validate_treeherder_data(parser)
if not self.run_local:
# copy results to upload dir so they are included as an artifact
dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
self._artifact_perf_data(dest)
self.buildbot_status(parser.worst_tbpl_status,
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -160,16 +160,19 @@ def create_parser(mach_interface=False):
add_arg('--framework',
help='Will post to the specified framework for Perfherder. '
'Default "talos". Used primarily for experiments on '
'new platforms')
add_arg('--print-tests', action=_ListTests,
help="print available tests")
add_arg('--print-suites', action=_ListSuite,
help="list available suites")
+ add_arg('--no-upload-results', action="store_true",
+ dest='no_upload_results',
+ help="If given, it disables uploading of talos results.")
add_logging_group(parser)
return parser
def parse_args(argv=None):
parser = create_parser()
return parser.parse_args(argv)
--- a/testing/talos/talos/config.py
+++ b/testing/talos/talos/config.py
@@ -426,16 +426,17 @@ def get_browser_config(config):
'process': '',
'framework': 'talos',
'repository': None,
'sourcestamp': None,
'symbols_path': None,
'test_timeout': 1200,
'xperf_path': None,
'error_filename': None,
+ 'no_upload_results': False,
}
browser_config = dict(title=config['title'])
browser_config.update(dict([(i, config[i]) for i in required]))
browser_config.update(dict([(i, config.get(i, j))
for i, j in optional.items()]))
return browser_config
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -282,17 +282,17 @@ def run_tests(config, browser_config):
LOG.info("Completed test suite (%s)" % timer.elapsed())
# if mitmproxy was used for page playback, stop it
if mitmproxy_recordings_list is not False:
mitmproxy.stop_mitmproxy_playback(mitmproxy_proc)
# output results
- if results_urls:
+ if results_urls and not browser_config['no_upload_results']:
talos_results.output(results_urls)
if browser_config['develop'] or config['gecko_profile']:
print("Thanks for running Talos locally. Results are in %s"
% (results_urls['output_urls']))
# we will stop running tests on a failed test, or we will return 0 for
# green
return 0