Bug 1476526 - Make the baseline test use functions from BrowserTestUtils. r?marco
This patch makes a new baseline coverage test for browser-chrome which makes use of the BrowserTestUtils functions. It also begins the necessary changes so that the baseline test addition uses suites instead of file types.
MozReview-Commit-ID: 7nucHnF8CIn
--- a/testing/mochitest/baselinecoverage/browser_chrome/browser.ini
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser.ini
@@ -1,4 +1,6 @@
[DEFAULT]
[browser_baselinecoverage.js]
+run-if = ccov && verify
+[browser_baselinecoverage_browser-chrome.js]
run-if = ccov && verify
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage_browser-chrome.js
@@ -0,0 +1,22 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+/* eslint-disable mozilla/no-arbitrary-setTimeout */
+
+"use strict";
+
+const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm", {});
+
+add_task(async function() {
+ requestLongerTimeout(2);
+ let newWin = await BrowserTestUtils.openNewBrowserWindow();
+
+ await BrowserTestUtils.withNewTab({
+ gBrowser,
+ url: "about:blank"
+ }, async function(browser) {
+ ok(true, "Collecting baseline coverage for browser-chrome tests.");
+ await new Promise((c) => setTimeout(c, 30 * 1000));
+ });
+
+ await BrowserTestUtils.closeWindow(newWin);
+});
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -149,52 +149,66 @@ class CodeCoverageMixin(SingleTestMixin)
if not self.per_test_coverage:
return
self.find_modified_tests()
# TODO: Add tests that haven't been run for a while (a week? N pushes?)
# Add baseline code coverage collection tests
- baseline_tests = {
+ baseline_tests_by_ext = {
'.html': {
'test': 'testing/mochitest/baselinecoverage/plain/test_baselinecoverage.html',
'suite': 'plain'
},
'.js': {
'test': 'testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage.js', # NOQA: E501
'suite': 'browser-chrome'
},
'.xul': {
'test': 'testing/mochitest/baselinecoverage/chrome/test_baselinecoverage.xul',
'suite': 'chrome'
}
}
+ baseline_tests_by_suite = {
+ 'browser-chrome': 'testing/mochitest/baselinecoverage/browser_chrome/'
+ 'browser_baselinecoverage_browser-chrome.js'
+ }
+
wpt_baseline_test = 'tests/web-platform/mozilla/tests/baselinecoverage/wpt_baselinecoverage.html' # NOQA: E501
if self.config.get('per_test_category') == "web-platform":
if 'testharness' not in self.suites:
self.suites['testharness'] = []
if wpt_baseline_test not in self.suites['testharness']:
self.suites["testharness"].append(wpt_baseline_test)
return
# Go through all the tests and find all
# the baseline tests that are needed.
tests_to_add = {}
for suite in self.suites:
+ if len(self.suites[suite]) == 0:
+ continue
+ if suite in baseline_tests_by_suite:
+ if suite not in tests_to_add:
+ tests_to_add[suite] = []
+ tests_to_add[suite].append(baseline_tests_by_suite[suite])
+ continue
+
+ # Default to file types if the suite has no baseline
for test in self.suites[suite]:
_, test_ext = os.path.splitext(test)
- if test_ext not in baseline_tests:
+ if test_ext not in baseline_tests_by_ext:
# Add the '.js' test as a default baseline
# if none other exists.
test_ext = '.js'
- baseline_test_suite = baseline_tests[test_ext]['suite']
- baseline_test_name = baseline_tests[test_ext]['test']
+ baseline_test_suite = baseline_tests_by_ext[test_ext]['suite']
+ baseline_test_name = baseline_tests_by_ext[test_ext]['test']
if baseline_test_suite not in tests_to_add:
tests_to_add[baseline_test_suite] = []
if baseline_test_name not in tests_to_add[baseline_test_suite]:
tests_to_add[baseline_test_suite].append(baseline_test_name)
# Add all baseline tests needed
for suite in tests_to_add:
@@ -357,28 +371,34 @@ class CodeCoverageMixin(SingleTestMixin)
return
if self.per_test_coverage:
if not self.per_test_reports:
self.info("No tests were found...not saving coverage data.")
return
# Get the baseline tests that were run.
- baseline_tests_cov = {}
+ baseline_tests_ext_cov = {}
+ baseline_tests_suite_cov = {}
for suite, data in self.per_test_reports.items():
for test, grcov_file in data.items():
if 'baselinecoverage' not in test:
continue
# TODO: Optimize this part which loads JSONs
# with a size of about 40Mb into memory for diffing later.
# Bug 1460064 is filed for this.
- _, baseline_filetype = os.path.splitext(test)
with open(grcov_file, 'r') as f:
- baseline_tests_cov[baseline_filetype] = json.load(f)
+ data = json.load(f)
+
+ if suite in test:
+ baseline_tests_suite_cov[suite] = data
+ else:
+ _, baseline_filetype = os.path.splitext(test)
+ baseline_tests_ext_cov[baseline_filetype] = data
dest = os.path.join(dirs['abs_blob_upload_dir'], 'per-test-coverage-reports.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED) as z:
for suite, data in self.per_test_reports.items():
for test, grcov_file in data.items():
if 'baselinecoverage' in test:
# Don't keep the baseline coverage
continue
@@ -389,29 +409,31 @@ class CodeCoverageMixin(SingleTestMixin)
# Remove uncovered files, as they are unneeded for per-test
# coverage purposes.
report['source_files'] = [
sf for sf in report['source_files'] if self.is_covered(sf)]
# Get baseline coverage
baseline_coverage = {}
- if self.config.get('per_test_category') == "web-platform":
- baseline_coverage = baseline_tests_cov['.html']
+ if suite in baseline_tests_suite_cov:
+ baseline_coverage = baseline_tests_suite_cov[suite]
+ elif self.config.get('per_test_category') == "web-platform":
+ baseline_coverage = baseline_tests_ext_cov['.html']
else:
- for file_type in baseline_tests_cov:
+ for file_type in baseline_tests_ext_cov:
if not test.endswith(file_type):
continue
- baseline_coverage = baseline_tests_cov[file_type]
+ baseline_coverage = baseline_tests_ext_cov[file_type]
break
if not baseline_coverage:
# Default to the '.js' baseline as it is the largest
self.info("Did not find a baseline test for: " + test)
- baseline_coverage = baseline_tests_cov['.js']
+ baseline_coverage = baseline_tests_ext_cov['.js']
unique_coverage = rm_baseline_cov(baseline_coverage, report)
with open(grcov_file, 'w') as f:
json.dump({
'test': test,
'suite': suite,
'report': unique_coverage,