Bug 1347241 - Enable linux64-ccov coverage collection for common tests set; r?jmaher, gmierz
MozReview-Commit-ID: FLyjCwMGb6L
--- a/devtools/client/memory/test/browser/browser.ini
+++ b/devtools/client/memory/test/browser/browser.ini
@@ -5,27 +5,30 @@ support-files =
head.js
doc_big_tree.html
doc_empty.html
doc_steady_allocation.html
!/devtools/client/framework/test/shared-head.js
!/devtools/client/framework/test/shared-redux-head.js
[browser_memory_allocationStackDisplay_01.js]
- skip-if = debug # bug 1219554
+skip-if = debug # bug 1219554
[browser_memory_displays_01.js]
[browser_memory_clear_snapshots.js]
[browser_memory_diff_01.js]
[browser_memory_dominator_trees_01.js]
+skip-if = coverage # bug 1347244
[browser_memory_dominator_trees_02.js]
+skip-if = coverage # bug 1347244
[browser_memory_filter_01.js]
+skip-if = coverage # bug 1347244
[browser_memory_individuals_01.js]
[browser_memory_keyboard.js]
[browser_memory_keyboard-snapshot-list.js]
[browser_memory_no_allocation_stacks.js]
[browser_memory_no_auto_expand.js]
- skip-if = debug # bug 1219554
+skip-if = debug # bug 1219554
[browser_memory_percents_01.js]
[browser_memory_refresh_does_not_leak.js]
[browser_memory_simple_01.js]
[browser_memory_transferHeapSnapshot_e10s_01.js]
[browser_memory_tree_map-01.js]
[browser_memory_tree_map-02.js]
--- a/js/src/jit-test/jit_test.py
+++ b/js/src/jit-test/jit_test.py
@@ -229,16 +229,23 @@ def main(argv):
sys.stderr.write("Exception thrown trying to read test file"
" '{}'\n".format(options.read_tests))
traceback.print_exc()
sys.stderr.write('---\n')
if read_all:
test_list = jittests.find_tests()
+ # If code coverage is enabled, exclude tests. (bug 1347245)
+ if os.getenv('GCOV_PREFIX') is not None:
+ if options.exclude:
+ options.exclude += ['asm.js/testSIMD.js']
+ else:
+ options.exclude = ['asm.js/testSIMD.js']
+
if options.exclude:
exclude_list = []
for exclude in options.exclude:
exclude_list += jittests.find_tests(exclude)
test_list = [test for test in test_list
if test not in set(exclude_list)]
if not test_list:
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -84,24 +84,36 @@ qr-tests:
- mochitest-a11y
- mochitest-gpu
- mochitest-webgl
- reftest
- reftest-no-accel
- xpcshell
ccov-code-coverage-tests:
+ - cppunit
- crashtest
+ - firefox-ui-functional-local
+ - firefox-ui-functional-remote
- gtest
+ - jittest
- jsreftest
- marionette
- mochitest
+ - mochitest-a11y
- mochitest-browser-chrome
+ - mochitest-chrome
+ - mochitest-clipboard
- mochitest-devtools-chrome
+ - mochitest-gpu
+ - mochitest-jetpack
+ - mochitest-media
+ - mochitest-webgl
- reftest
+ - reftest-no-accel
- web-platform-tests
- xpcshell
jsdcov-code-coverage-tests:
- mochitest
- mochitest-browser-chrome
- mochitest-devtools-chrome
- xpcshell
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -4,16 +4,20 @@
# for a description of the fields used here.
# Note that these are in lexical order
cppunit:
description: "CPP Unit Tests"
suite: cppunittest
treeherder-symbol: tc(Cpp)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
linux64-qr/.*: ['graphics']
linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
default: ['all']
@@ -165,35 +169,51 @@ external-media-tests-youtube:
- remove_executables.py
extra-options:
- "--suite=media-youtube-tests"
firefox-ui-functional-local:
description: "Firefox-ui-tests functional run"
suite: "firefox-ui/functional local"
treeherder-symbol: tc-Fxfn-l(en-US)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
max-run-time: 5400
tier: 1
docker-image: {"in-tree": "desktop1604-test"}
+ e10s:
+ by-test-platform:
+ linux64-ccov/opt: false
+ default: both
mozharness:
script: firefox_ui_tests/functional.py
config:
- firefox_ui_tests/taskcluster.py
- remove_executables.py
extra-options:
- "--tag"
- "local"
firefox-ui-functional-remote:
description: "Firefox-ui-tests functional run"
suite: "firefox-ui/functional remote"
treeherder-symbol: tc-Fxfn-r(en-US)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
max-run-time: 5400
tier: 2
docker-image: {"in-tree": "desktop1604-test"}
+ e10s:
+ by-test-platform:
+ linux64-ccov/opt: false
+ default: both
mozharness:
script: firefox_ui_tests/functional.py
config:
- firefox_ui_tests/taskcluster.py
- remove_executables.py
extra-options:
- "--tag"
- "remote"
@@ -224,16 +244,20 @@ gtest:
- remove_executables.py
extra-options:
- --gtest-suite=gtest
jittest:
description: "JIT Test run"
suite: jittest/jittest-chunked
treeherder-symbol: tc(Jit)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
linux64-qr/.*: ['graphics']
default: ['all']
chunks:
by-test-platform:
@@ -440,16 +464,20 @@ mochitest:
- --mochitest-suite=plain-chunked-coverage
default:
- --mochitest-suite=plain-chunked
mochitest-a11y:
description: "Mochitest a11y run"
suite: mochitest/a11y
treeherder-symbol: tc-M(a11y)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
loopback-video: true
e10s: false
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
linux64-qr/.*: ['graphics']
default: ['all']
mozharness:
@@ -560,16 +588,17 @@ mochitest-browser-screenshots:
mochitest-chrome:
description: "Mochitest chrome run"
suite: mochitest/chrome
treeherder-symbol: tc-M(c)
loopback-video: true
instance-size:
by-test-platform:
android.*: xlarge
+ linux64-ccov/opt: xlarge
default: default
chunks:
by-test-platform:
android-4.3-arm7-api-15/debug: 4
android.*: 2
default: 3
max-run-time: 3600
e10s: false
@@ -609,16 +638,17 @@ mochitest-clipboard:
description: "Mochitest clipboard run"
suite: mochitest/clipboard
treeherder-symbol: tc-M(cl)
loopback-video: true
instance-size: xlarge
e10s:
by-test-platform:
macosx64/debug: true
+ linux64-ccov/opt: false
default: both
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
@@ -698,27 +728,32 @@ mochitest-devtools-chrome:
default: default
# Bug 1296086: high number of intermittents observed with software GL and large instances
allow-software-gl-layers: false
mochitest-gpu:
description: "Mochitest GPU run"
suite: mochitest/gpu
treeherder-symbol: tc-M(gpu)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
loopback-video: true
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
linux64-qr/.*: ['graphics']
default: ['all']
e10s:
by-test-platform:
windows.*: both
android.*: false
macosx64/opt: both
+ linux64-ccov/opt: false
default: true
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
no-read-buildbot-config: true
config:
@@ -743,16 +778,20 @@ mochitest-gpu:
- remove_executables.py
extra-options:
- --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
mochitest-jetpack:
description: "Mochitest jetpack run"
suite: mochitest/jetpack-package
treeherder-symbol: tc-M(JP)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
loopback-video: true
e10s: false
max-run-time: 5400
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
windows.*: ['mozilla-central', 'try']
default: ['all']
@@ -779,20 +818,22 @@ mochitest-media:
description: "Mochitest media run"
suite: mochitest/mochitest-media
treeherder-symbol: tc-M(mda)
max-run-time: 5400
loopback-video: true
e10s:
by-test-platform:
macosx64/debug: true
+ linux64-ccov/opt: false
default: both
instance-size:
by-test-platform:
android.*: xlarge
+ linux64-ccov/opt: xlarge
default: large
chunks:
by-test-platform:
android.*: 3
macosx64.*: 1
default: 3
docker-image: {"in-tree": "desktop1604-test"}
mozharness:
@@ -864,25 +905,27 @@ mochitest-webgl:
default: ['all']
chunks:
by-test-platform:
android.*: 10
default: 3
e10s:
by-test-platform:
macosx.*: true
+ linux64-ccov/opt: false
default: both
loopback-video: true
max-run-time:
by-test-platform:
android.*: 7200
default: 3600
instance-size:
by-test-platform:
android.*: xlarge
+ linux64-ccov/opt: xlarge
default: default
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
by-test-platform:
android.*:
mochitest-flavor: plain
script: android_emulator_unittest.py
@@ -1010,25 +1053,33 @@ reftest:
- remove_executables.py
extra-options:
- --reftest-suite=reftest
reftest-no-accel:
description: "Reftest not accelerated run"
suite: reftest/reftest-no-accel
treeherder-symbol: tc-R(Ru)
+ instance-size:
+ by-test-platform:
+ linux64-ccov/opt: xlarge
+ default: default
docker-image: {"in-tree": "desktop1604-test"}
run-on-projects:
by-test-platform:
linux64-qr/.*: ['graphics', 'mozilla-central']
default: ['all']
chunks:
- by-test-platform:
- macosx.*: 1
- default: 8
+ by-test-platform:
+ macosx.*: 1
+ default: 8
+ e10s:
+ by-test-platform:
+ linux64-ccov/opt: false
+ default: both
mozharness:
script: desktop_unittest.py
no-read-buildbot-config: true
config:
by-test-platform:
windows.*:
- unittests/win_taskcluster_unittest.py
macosx.*:
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2273,16 +2273,20 @@ toolbar#nav-bar {
""" Prepare, configure, run tests and cleanup """
# a11y and chrome tests don't run with e10s enabled in CI. Need to set
# this here since |mach mochitest| sets the flavor after argument parsing.
if options.flavor in ('a11y', 'chrome'):
options.e10s = False
mozinfo.update({"e10s": options.e10s}) # for test manifest parsing.
+ # Add flag to mozinfo to indicate that code coverage is enabled.
+ if os.getenv('GCOV_PREFIX') is not None:
+ mozinfo.update({"coverage": True})
+
self.setTestRoot(options)
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
# Try to avoid those failures by checking for and killing orphan servers before
# trying to start new ones.
self.killNamedOrphans('ssltunnel')
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
@@ -12,16 +12,20 @@ import sys
from mozharness.base.log import FATAL, WARNING
from mozharness.base.python import PostScriptRun, PreScriptAction
from mozharness.mozilla.structuredlog import StructuredOutputParser
from mozharness.mozilla.testing.testbase import (
TestingMixin,
testing_config_options,
)
+from mozharness.mozilla.testing.codecoverage import (
+ CodeCoverageMixin,
+ code_coverage_config_options
+)
from mozharness.mozilla.vcstools import VCSToolsScript
# General command line arguments for Firefox ui tests
firefox_ui_tests_config_options = [
[["--allow-software-gl-layers"], {
"action": "store_true",
"dest": "allow_software_gl_layers",
@@ -43,17 +47,18 @@ firefox_ui_tests_config_options = [
'dest': 'symbols_path',
'help': 'absolute path to directory containing breakpad '
'symbols, or the url of a zip file containing symbols.',
}],
[['--tag=TAG'], {
'dest': 'tag',
'help': 'Subset of tests to run (local, remote).',
}],
-] + copy.deepcopy(testing_config_options)
+] + copy.deepcopy(testing_config_options) \
+ + copy.deepcopy(code_coverage_config_options)
# Command line arguments for update tests
firefox_ui_update_harness_config_options = [
[['--update-allow-mar-channel'], {
'dest': 'update_allow_mar_channel',
'help': 'Additional MAR channel to be allowed for updates, e.g. '
'"firefox-mozilla-beta" for updating a release build to '
'the latest beta build.',
@@ -85,17 +90,17 @@ firefox_ui_update_harness_config_options
'help': 'Version of the updated build.',
}],
]
firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
+ copy.deepcopy(firefox_ui_tests_config_options)
-class FirefoxUITests(TestingMixin, VCSToolsScript):
+class FirefoxUITests(TestingMixin, VCSToolsScript, CodeCoverageMixin):
# Needs to be overwritten in sub classes
cli_script = None
def __init__(self, config_options=None,
all_actions=None, default_actions=None,
*args, **kwargs):
config_options = config_options or firefox_ui_tests_config_options
@@ -238,16 +243,20 @@ class FirefoxUITests(TestingMixin, VCSTo
# Set further environment settings
env = env or self.query_env()
env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']})
if self.query_minidump_stackwalk():
env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path})
env['RUST_BACKTRACE'] = '1'
+ # If code coverage is enabled, set GCOV_PREFIX env variable
+ if self.config.get('code_coverage'):
+ env['GCOV_PREFIX'] = self.gcov_dir
+
if self.config['allow_software_gl_layers']:
env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
return_code = self.run_command(cmd,
cwd=dirs['abs_work_dir'],
output_timeout=300,
output_parser=parser,
env=env)