Bug 1347241 - Enable linux64-ccov coverage collection for common tests set; r?jmaher, gmierz draft collect-ccov-common-tests
authorMadeleine Chercover <madeleinechercover@me.com>
Tue, 14 Mar 2017 11:40:59 -0700
branchcollect-ccov-common-tests
changeset 498634 35f1fa44baae75c9aaaae4da47a28c725371be47
parent 498360 2baef2ffbaedb7354286726660ebd36e84b432f0
child 724648 385d32c7eeea2568600190d4090c2a130919ffb5
push id49251
push userbmo:madeleinechercover@me.com
push dateWed, 15 Mar 2017 02:27:07 +0000
reviewersjmaher, gmierz
bugs1347241
milestone55.0a1
Bug 1347241 - Enable linux64-ccov coverage collection for common tests set; r?jmaher, gmierz MozReview-Commit-ID: FLyjCwMGb6L
devtools/client/memory/test/browser/browser.ini
js/src/jit-test/jit_test.py
taskcluster/ci/test/test-sets.yml
taskcluster/ci/test/tests.yml
testing/mochitest/runtests.py
testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
--- a/devtools/client/memory/test/browser/browser.ini
+++ b/devtools/client/memory/test/browser/browser.ini
@@ -5,27 +5,30 @@ support-files =
   head.js
   doc_big_tree.html
   doc_empty.html
   doc_steady_allocation.html
   !/devtools/client/framework/test/shared-head.js
   !/devtools/client/framework/test/shared-redux-head.js
 
 [browser_memory_allocationStackDisplay_01.js]
-    skip-if = debug # bug 1219554
+skip-if = debug # bug 1219554
 [browser_memory_displays_01.js]
 [browser_memory_clear_snapshots.js]
 [browser_memory_diff_01.js]
 [browser_memory_dominator_trees_01.js]
+skip-if = coverage # bug 1347244
 [browser_memory_dominator_trees_02.js]
+skip-if = coverage # bug 1347244
 [browser_memory_filter_01.js]
+skip-if = coverage # bug 1347244
 [browser_memory_individuals_01.js]
 [browser_memory_keyboard.js]
 [browser_memory_keyboard-snapshot-list.js]
 [browser_memory_no_allocation_stacks.js]
 [browser_memory_no_auto_expand.js]
-    skip-if = debug # bug 1219554
+skip-if = debug # bug 1219554
 [browser_memory_percents_01.js]
 [browser_memory_refresh_does_not_leak.js]
 [browser_memory_simple_01.js]
 [browser_memory_transferHeapSnapshot_e10s_01.js]
 [browser_memory_tree_map-01.js]
 [browser_memory_tree_map-02.js]
--- a/js/src/jit-test/jit_test.py
+++ b/js/src/jit-test/jit_test.py
@@ -229,16 +229,23 @@ def main(argv):
                 sys.stderr.write("Exception thrown trying to read test file"
                                  " '{}'\n".format(options.read_tests))
                 traceback.print_exc()
                 sys.stderr.write('---\n')
 
     if read_all:
         test_list = jittests.find_tests()
 
+    # If code coverage is enabled, exclude tests. (bug 1347245)
+    if os.getenv('GCOV_PREFIX') is not None:
+        if options.exclude:
+            options.exclude += ['asm.js/testSIMD.js']
+        else:
+            options.exclude = ['asm.js/testSIMD.js']
+
     if options.exclude:
         exclude_list = []
         for exclude in options.exclude:
             exclude_list += jittests.find_tests(exclude)
         test_list = [test for test in test_list
                      if test not in set(exclude_list)]
 
     if not test_list:
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -84,24 +84,36 @@ qr-tests:
     - mochitest-a11y
     - mochitest-gpu
     - mochitest-webgl
     - reftest
     - reftest-no-accel
     - xpcshell
 
 ccov-code-coverage-tests:
+    - cppunit
     - crashtest
+    - firefox-ui-functional-local
+    - firefox-ui-functional-remote
     - gtest
+    - jittest
     - jsreftest
     - marionette
     - mochitest
+    - mochitest-a11y
     - mochitest-browser-chrome
+    - mochitest-chrome
+    - mochitest-clipboard
     - mochitest-devtools-chrome
+    - mochitest-gpu
+    - mochitest-jetpack
+    - mochitest-media
+    - mochitest-webgl
     - reftest
+    - reftest-no-accel
     - web-platform-tests
     - xpcshell
 
 jsdcov-code-coverage-tests:
     - mochitest
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
     - xpcshell
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -4,16 +4,20 @@
 # for a description of the fields used here.
 
 # Note that these are in lexical order
 
 cppunit:
     description: "CPP Unit Tests"
     suite: cppunittest
     treeherder-symbol: tc(Cpp)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
             linux64-qr/.*: ['graphics']
             linux64-stylo/.*: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central', 'try' ]
             default: ['all']
@@ -165,35 +169,51 @@ external-media-tests-youtube:
                     - remove_executables.py
         extra-options:
             - "--suite=media-youtube-tests"
 
 firefox-ui-functional-local:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional local"
     treeherder-symbol: tc-Fxfn-l(en-US)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     max-run-time: 5400
     tier: 1
     docker-image: {"in-tree": "desktop1604-test"}
+    e10s:
+        by-test-platform:
+            linux64-ccov/opt: false
+            default: both
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
             - "--tag"
             - "local"
 
 firefox-ui-functional-remote:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional remote"
     treeherder-symbol: tc-Fxfn-r(en-US)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     max-run-time: 5400
     tier: 2
     docker-image: {"in-tree": "desktop1604-test"}
+    e10s:
+        by-test-platform:
+            linux64-ccov/opt: false
+            default: both
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
             - "--tag"
             - "remote"
@@ -224,16 +244,20 @@ gtest:
                     - remove_executables.py
         extra-options:
             - --gtest-suite=gtest
 
 jittest:
     description: "JIT Test run"
     suite: jittest/jittest-chunked
     treeherder-symbol: tc(Jit)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             linux64-qr/.*: ['graphics']
             default: ['all']
     chunks:
         by-test-platform:
@@ -440,16 +464,20 @@ mochitest:
                             - --mochitest-suite=plain-chunked-coverage
                         default:
                             - --mochitest-suite=plain-chunked
 
 mochitest-a11y:
     description: "Mochitest a11y run"
     suite: mochitest/a11y
     treeherder-symbol: tc-M(a11y)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     loopback-video: true
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             linux64-qr/.*: ['graphics']
             default: ['all']
     mozharness:
@@ -560,16 +588,17 @@ mochitest-browser-screenshots:
 mochitest-chrome:
     description: "Mochitest chrome run"
     suite: mochitest/chrome
     treeherder-symbol: tc-M(c)
     loopback-video: true
     instance-size:
         by-test-platform:
             android.*: xlarge
+            linux64-ccov/opt: xlarge
             default: default
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 4
             android.*: 2
             default: 3
     max-run-time: 3600
     e10s: false
@@ -609,16 +638,17 @@ mochitest-clipboard:
     description: "Mochitest clipboard run"
     suite: mochitest/clipboard
     treeherder-symbol: tc-M(cl)
     loopback-video: true
     instance-size: xlarge
     e10s:
       by-test-platform:
         macosx64/debug: true
+        linux64-ccov/opt: false
         default: both
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
@@ -698,27 +728,32 @@ mochitest-devtools-chrome:
             default: default
     # Bug 1296086: high number of intermittents observed with software GL and large instances
     allow-software-gl-layers: false
 
 mochitest-gpu:
     description: "Mochitest GPU run"
     suite: mochitest/gpu
     treeherder-symbol: tc-M(gpu)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     loopback-video: true
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             linux64-qr/.*: ['graphics']
             default: ['all']
     e10s:
         by-test-platform:
             windows.*: both
             android.*: false
             macosx64/opt: both
+            linux64-ccov/opt: false
             default: true
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
@@ -743,16 +778,20 @@ mochitest-gpu:
                             - remove_executables.py
                 extra-options:
                     - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
 
 mochitest-jetpack:
     description: "Mochitest jetpack run"
     suite: mochitest/jetpack-package
     treeherder-symbol: tc-M(JP)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     loopback-video: true
     e10s: false
     max-run-time: 5400
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             windows.*: ['mozilla-central', 'try']
             default: ['all']
@@ -779,20 +818,22 @@ mochitest-media:
     description: "Mochitest media run"
     suite: mochitest/mochitest-media
     treeherder-symbol: tc-M(mda)
     max-run-time: 5400
     loopback-video: true
     e10s:
       by-test-platform:
         macosx64/debug: true
+        linux64-ccov/opt: false
         default: both
     instance-size:
         by-test-platform:
             android.*: xlarge
+            linux64-ccov/opt: xlarge
             default: large
     chunks:
         by-test-platform:
             android.*: 3
             macosx64.*: 1
             default: 3
     docker-image: {"in-tree": "desktop1604-test"}
     mozharness:
@@ -864,25 +905,27 @@ mochitest-webgl:
             default: ['all']
     chunks:
         by-test-platform:
             android.*: 10
             default: 3
     e10s:
       by-test-platform:
         macosx.*: true
+        linux64-ccov/opt: false
         default: both
     loopback-video: true
     max-run-time:
         by-test-platform:
             android.*: 7200
             default: 3600
     instance-size:
         by-test-platform:
             android.*: xlarge
+            linux64-ccov/opt: xlarge
             default: default
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         by-test-platform:
             android.*:
                 mochitest-flavor: plain
                 script: android_emulator_unittest.py
@@ -1010,25 +1053,33 @@ reftest:
                             - remove_executables.py
                 extra-options:
                     - --reftest-suite=reftest
 
 reftest-no-accel:
     description: "Reftest not accelerated run"
     suite: reftest/reftest-no-accel
     treeherder-symbol: tc-R(Ru)
+    instance-size:
+        by-test-platform:
+            linux64-ccov/opt: xlarge
+            default: default
     docker-image: {"in-tree": "desktop1604-test"}
     run-on-projects:
         by-test-platform:
             linux64-qr/.*: ['graphics', 'mozilla-central']
             default: ['all']
     chunks:
-      by-test-platform:
-        macosx.*: 1
-        default: 8
+        by-test-platform:
+            macosx.*: 1
+            default: 8
+    e10s:
+        by-test-platform:
+            linux64-ccov/opt: false
+            default: both
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2273,16 +2273,20 @@ toolbar#nav-bar {
         """ Prepare, configure, run tests and cleanup """
 
         # a11y and chrome tests don't run with e10s enabled in CI. Need to set
         # this here since |mach mochitest| sets the flavor after argument parsing.
         if options.flavor in ('a11y', 'chrome'):
             options.e10s = False
         mozinfo.update({"e10s": options.e10s})  # for test manifest parsing.
 
+        # Add flag to mozinfo to indicate that code coverage is enabled.
+        if os.getenv('GCOV_PREFIX') is not None:
+            mozinfo.update({"coverage": True})
+
         self.setTestRoot(options)
 
         # Despite our efforts to clean up servers started by this script, in practice
         # we still see infrequent cases where a process is orphaned and interferes
         # with future tests, typically because the old server is keeping the port in use.
         # Try to avoid those failures by checking for and killing orphan servers before
         # trying to start new ones.
         self.killNamedOrphans('ssltunnel')
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
@@ -12,16 +12,20 @@ import sys
 
 from mozharness.base.log import FATAL, WARNING
 from mozharness.base.python import PostScriptRun, PreScriptAction
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.testbase import (
     TestingMixin,
     testing_config_options,
 )
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
 from mozharness.mozilla.vcstools import VCSToolsScript
 
 
 # General command line arguments for Firefox ui tests
 firefox_ui_tests_config_options = [
     [["--allow-software-gl-layers"], {
         "action": "store_true",
         "dest": "allow_software_gl_layers",
@@ -43,17 +47,18 @@ firefox_ui_tests_config_options = [
         'dest': 'symbols_path',
         'help': 'absolute path to directory containing breakpad '
                 'symbols, or the url of a zip file containing symbols.',
     }],
     [['--tag=TAG'], {
         'dest': 'tag',
         'help': 'Subset of tests to run (local, remote).',
     }],
-] + copy.deepcopy(testing_config_options)
+] + copy.deepcopy(testing_config_options) \
+  + copy.deepcopy(code_coverage_config_options)
 
 # Command line arguments for update tests
 firefox_ui_update_harness_config_options = [
     [['--update-allow-mar-channel'], {
         'dest': 'update_allow_mar_channel',
         'help': 'Additional MAR channel to be allowed for updates, e.g. '
                 '"firefox-mozilla-beta" for updating a release build to '
                 'the latest beta build.',
@@ -85,17 +90,17 @@ firefox_ui_update_harness_config_options
         'help': 'Version of the updated build.',
     }],
 ]
 
 firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
     + copy.deepcopy(firefox_ui_tests_config_options)
 
 
-class FirefoxUITests(TestingMixin, VCSToolsScript):
+class FirefoxUITests(TestingMixin, VCSToolsScript, CodeCoverageMixin):
 
     # Needs to be overwritten in sub classes
     cli_script = None
 
     def __init__(self, config_options=None,
                  all_actions=None, default_actions=None,
                  *args, **kwargs):
         config_options = config_options or firefox_ui_tests_config_options
@@ -238,16 +243,20 @@ class FirefoxUITests(TestingMixin, VCSTo
 
         # Set further environment settings
         env = env or self.query_env()
         env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']})
         if self.query_minidump_stackwalk():
             env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path})
         env['RUST_BACKTRACE'] = '1'
 
+        # If code coverage is enabled, set GCOV_PREFIX env variable
+        if self.config.get('code_coverage'):
+            env['GCOV_PREFIX'] = self.gcov_dir
+
         if self.config['allow_software_gl_layers']:
             env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
 
         return_code = self.run_command(cmd,
                                        cwd=dirs['abs_work_dir'],
                                        output_timeout=300,
                                        output_parser=parser,
                                        env=env)