Bug 1455401 - Generate baseline code coverage reports. r?marco,jmaher
The tests added in this patch will be used as a baseline for coverage that is common to all tests. They are added to each chunk being run in the test-coverage suite only if a test with an associated baseline type exists.
MozReview-Commit-ID: 1CrRZ1Ev2Mz
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/browser_chrome/.eslintrc.js
@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+ "extends": [
+ "plugin:mozilla/browser-test"
+ ]
+};
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+
+[browser_baselinecoverage.js]
+run-if = ccov && verify
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage.js
@@ -0,0 +1,8 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function() {
+ ok(true, "Collecting baseline coverage for javascript (.js) file types.");
+});
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/chrome/chrome.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+
+[test_baselinecoverage.xul]
+run-if = ccov && verify
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/chrome/test_baselinecoverage.xul
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+ - License, v. 2.0. If a copy of the MPL was not distributed with this
+ - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<?xml-stylesheet href="chrome://global/skin/" type="text/css"?>
+<?xml-stylesheet href="chrome://mochikit/content/tests/SimpleTest/test.css" type="text/css"?>
+
+<window xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+ <script type="application/javascript"
+ src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js" />
+ <script type="application/javascript"><![CDATA[
+ SimpleTest.ok(true, "Collecting baseline coverage for XUL (.xul) file types.");
+ ]]></script>
+
+ <body xmlns="http://www.w3.org/1999/xhtml" style="height: 300px; overflow: auto;" />
+</window>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/plain/.eslintrc.js
@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+ "extends": [
+ "plugin:mozilla/mochitest-test"
+ ]
+};
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/plain/mochitest.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+
+[test_baselinecoverage.html]
+run-if = ccov && verify
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/plain/test_baselinecoverage.html
@@ -0,0 +1,13 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+ <title>Baseline Coverage Collection</title>
+ <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<script type="application/javascript">
+ SimpleTest.ok(true, "Collecting baseline coverage for HTML (.html) file types.");
+</script>
+</body>
+</html>
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@@ -17,19 +17,28 @@ JAR_MANIFESTS += ['jar.mn']
USE_EXTENSION_MANIFEST = True
FINAL_TARGET_PP_FILES += ['install.rdf']
FINAL_TARGET_FILES += ['bootstrap.js']
MOCHITEST_MANIFESTS += [
+ 'baselinecoverage/plain/mochitest.ini',
'tests/MochiKit-1.4.2/tests/mochitest.ini',
]
-MOCHITEST_CHROME_MANIFESTS += ['chrome/chrome.ini']
+
+MOCHITEST_CHROME_MANIFESTS += [
+ 'baselinecoverage/chrome/chrome.ini',
+ 'chrome/chrome.ini'
+]
+
+BROWSER_CHROME_MANIFESTS += [
+ 'baselinecoverage/browser_chrome/browser.ini'
+]
TEST_HARNESS_FILES.testing.mochitest += [
'/build/mobile/remoteautomation.py',
'/build/pgo/server-locations.txt',
'/build/sanitizers/lsan_suppressions.txt',
'/build/sanitizers/ubsan_suppressions.txt',
'/build/valgrind/cross-architecture.sup',
'/build/valgrind/i386-pc-linux-gnu.sup',
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -162,18 +162,61 @@ class CodeCoverageMixin(SingleTestMixin)
This depends on test manifests, so can only run after test zips have
been downloaded and extracted.
"""
if not self.per_test_coverage:
return
self.find_modified_tests()
+
# TODO: Add tests that haven't been run for a while (a week? N pushes?)
+ # Add baseline code coverage collection tests
+ baseline_tests = {
+ '.html': {
+ 'test': 'testing/mochitest/baselinecoverage/plain/test_baselinecoverage.html',
+ 'suite': 'plain'
+ },
+ '.js': {
+ 'test': 'testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage.js',
+ 'suite': 'browser-chrome'
+ },
+ '.xul': {
+ 'test': 'testing/mochitest/baselinecoverage/chrome/test_baselinecoverage.xul',
+ 'suite': 'chrome'
+ }
+ }
+
+ wpt_baseline_test = 'tests/web-platform/mozilla/tests/baselinecoverage/wpt_baselinecoverage.html'
+ if self.config.get('per_test_category') == "web-platform":
+ if 'testharness' not in self.suites:
+ self.suites['testharness'] = []
+ if wpt_baseline_test not in self.suites['testharness']:
+ self.suites["testharness"].append(wpt_baseline_test)
+ return
+
+ # Go through all the tests and add all
+ # the baseline tests that are needed.
+ for suite in self.suites:
+ for test in self.suites[suite]:
+ _, test_ext = os.path.splitext(test)
+
+ if test_ext not in baseline_tests:
+ # Add the '.js' test as a default baseline
+ # if none other exists.
+ test_ext = '.js'
+ baseline_test_suite = baseline_tests[test_ext]['suite']
+ baseline_test_name = baseline_tests[test_ext]['test']
+
+ if baseline_test_suite not in self.suites:
+ self.suites[baseline_test_suite] = []
+ if baseline_test_name not in self.suites[baseline_test_suite]:
+ self.suites[baseline_test_suite].append(baseline_test_name)
+
@property
def coverage_args(self):
return []
def set_coverage_env(self, env):
# Set the GCOV directory.
gcov_dir = tempfile.mkdtemp()
env['GCOV_PREFIX'] = gcov_dir
--- a/testing/web-platform/mozilla/meta/MANIFEST.json
+++ b/testing/web-platform/mozilla/meta/MANIFEST.json
@@ -410,16 +410,22 @@
],
"wasm/js/unwind.wast.js": [
[
{}
]
]
},
"testharness": {
+ "baselinecoverage/wpt_baselinecoverage.html": [
+ [
+ "/_mozilla/baselinecoverage/wpt_baselinecoverage.html",
+ {}
+ ]
+ ],
"dom/classList.html": [
[
"/_mozilla/dom/classList.html",
{}
]
],
"dom/throttling/throttling-1.window.js": [
[
@@ -987,16 +993,20 @@
]
}
},
"paths": {
"./placeholder": [
"74e16eb87ecdfeb2dfc28f36e0c73a584abdf9c2",
"support"
],
+ "baselinecoverage/wpt_baselinecoverage.html": [
+ "8deca29e7b82fc7ef4c11d2a16e8611dfb8cf487",
+ "testharness"
+ ],
"dom/classList.html": [
"17c65652a2943738a0cc5152aec737f243a02d3d",
"testharness"
],
"dom/throttling/resources/test.html": [
"0898e3efa5dbcc74138bdf27ff2787d1f1a2d4d8",
"support"
],
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mozilla/meta/baselinecoverage/wpt_baselinecoverage.html.ini
@@ -0,0 +1,3 @@
+[wpt_baselinecoverage.html]
+ disabled:
+ if not ccov: Code-Coverage-Test
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mozilla/tests/baselinecoverage/wpt_baselinecoverage.html
@@ -0,0 +1,10 @@
+<!doctype html>
+<meta charset=utf-8>
+<title>Baseline Coverage</title>
+<script src=/resources/testharness.js></script>
+<script src=/resources/testharnessreport.js></script>
+<script>
+ test(function() {
+ assert_equals(true, true);
+ }, "Collecting baseline coverage for web-platform tests.");
+</script>