Bug 1476526 - Make the baseline test use functions from BrowserTestUtils. r?marco draft
authorGreg Mierzwinski <gmierz1@live.ca>
Mon, 30 Jul 2018 10:20:48 -0400
changeset 826301 c0299c13984fb8e5b285be28027ae2e282c3d47f
parent 824743 5a5fb40fb92245de198f4fc48c1187a2b5abd02a
push id118280
push userbmo:gmierz2@outlook.com
push dateFri, 03 Aug 2018 12:47:39 +0000
reviewersmarco
bugs1476526
milestone63.0a1
Bug 1476526 - Make the baseline test use functions from BrowserTestUtils. r?marco This patch makes a new baseline coverage test for browser-chrome which makes use of the BrowserTestUtils functions. It also begins the necessary changes so that the baseline test addition uses suites instead of file types. MozReview-Commit-ID: 7nucHnF8CIn
testing/mochitest/baselinecoverage/browser_chrome/browser.ini
testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage_browser-chrome.js
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
--- a/testing/mochitest/baselinecoverage/browser_chrome/browser.ini
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser.ini
@@ -1,4 +1,6 @@
 [DEFAULT]
 
 [browser_baselinecoverage.js]
+run-if = ccov && verify
+[browser_baselinecoverage_browser-chrome.js]
 run-if = ccov && verify
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage_browser-chrome.js
@@ -0,0 +1,22 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+/* eslint-disable mozilla/no-arbitrary-setTimeout */
+
+"use strict";
+
+const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm", {});
+
+add_task(async function() {
+  requestLongerTimeout(2);
+  let newWin = await BrowserTestUtils.openNewBrowserWindow();
+
+  await BrowserTestUtils.withNewTab({
+    gBrowser,
+    url: "about:blank"
+  }, async function(browser) {
+    ok(true, "Collecting baseline coverage for browser-chrome tests.");
+    await new Promise((c) => setTimeout(c, 30 * 1000));
+  });
+
+  await BrowserTestUtils.closeWindow(newWin);
+});
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -149,52 +149,66 @@ class CodeCoverageMixin(SingleTestMixin)
         if not self.per_test_coverage:
             return
 
         self.find_modified_tests()
 
         # TODO: Add tests that haven't been run for a while (a week? N pushes?)
 
         # Add baseline code coverage collection tests
-        baseline_tests = {
+        baseline_tests_by_ext = {
             '.html': {
                 'test': 'testing/mochitest/baselinecoverage/plain/test_baselinecoverage.html',
                 'suite': 'plain'
             },
             '.js': {
                 'test': 'testing/mochitest/baselinecoverage/browser_chrome/browser_baselinecoverage.js',  # NOQA: E501
                 'suite': 'browser-chrome'
             },
             '.xul': {
                 'test': 'testing/mochitest/baselinecoverage/chrome/test_baselinecoverage.xul',
                 'suite': 'chrome'
             }
         }
 
+        baseline_tests_by_suite = {
+            'browser-chrome': 'testing/mochitest/baselinecoverage/browser_chrome/'
+                              'browser_baselinecoverage_browser-chrome.js'
+        }
+
         wpt_baseline_test = 'tests/web-platform/mozilla/tests/baselinecoverage/wpt_baselinecoverage.html'  # NOQA: E501
         if self.config.get('per_test_category') == "web-platform":
             if 'testharness' not in self.suites:
                 self.suites['testharness'] = []
             if wpt_baseline_test not in self.suites['testharness']:
                 self.suites["testharness"].append(wpt_baseline_test)
             return
 
         # Go through all the tests and find all
         # the baseline tests that are needed.
         tests_to_add = {}
         for suite in self.suites:
+            if len(self.suites[suite]) == 0:
+                continue
+            if suite in baseline_tests_by_suite:
+                if suite not in tests_to_add:
+                    tests_to_add[suite] = []
+                tests_to_add[suite].append(baseline_tests_by_suite[suite])
+                continue
+
+            # Default to file types if the suite has no baseline
             for test in self.suites[suite]:
                 _, test_ext = os.path.splitext(test)
 
-                if test_ext not in baseline_tests:
+                if test_ext not in baseline_tests_by_ext:
                     # Add the '.js' test as a default baseline
                     # if none other exists.
                     test_ext = '.js'
-                baseline_test_suite = baseline_tests[test_ext]['suite']
-                baseline_test_name = baseline_tests[test_ext]['test']
+                baseline_test_suite = baseline_tests_by_ext[test_ext]['suite']
+                baseline_test_name = baseline_tests_by_ext[test_ext]['test']
 
                 if baseline_test_suite not in tests_to_add:
                     tests_to_add[baseline_test_suite] = []
                 if baseline_test_name not in tests_to_add[baseline_test_suite]:
                     tests_to_add[baseline_test_suite].append(baseline_test_name)
 
         # Add all baseline tests needed
         for suite in tests_to_add:
@@ -357,28 +371,34 @@ class CodeCoverageMixin(SingleTestMixin)
             return
 
         if self.per_test_coverage:
             if not self.per_test_reports:
                 self.info("No tests were found...not saving coverage data.")
                 return
 
             # Get the baseline tests that were run.
-            baseline_tests_cov = {}
+            baseline_tests_ext_cov = {}
+            baseline_tests_suite_cov = {}
             for suite, data in self.per_test_reports.items():
                 for test, grcov_file in data.items():
                     if 'baselinecoverage' not in test:
                         continue
 
                     # TODO: Optimize this part which loads JSONs
                     # with a size of about 40Mb into memory for diffing later.
                     # Bug 1460064 is filed for this.
-                    _, baseline_filetype = os.path.splitext(test)
                     with open(grcov_file, 'r') as f:
-                        baseline_tests_cov[baseline_filetype] = json.load(f)
+                        data = json.load(f)
+
+                    if suite in test:
+                        baseline_tests_suite_cov[suite] = data
+                    else:
+                        _, baseline_filetype = os.path.splitext(test)
+                        baseline_tests_ext_cov[baseline_filetype] = data
 
             dest = os.path.join(dirs['abs_blob_upload_dir'], 'per-test-coverage-reports.zip')
             with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED) as z:
                 for suite, data in self.per_test_reports.items():
                     for test, grcov_file in data.items():
                         if 'baselinecoverage' in test:
                             # Don't keep the baseline coverage
                             continue
@@ -389,29 +409,31 @@ class CodeCoverageMixin(SingleTestMixin)
 
                             # Remove uncovered files, as they are unneeded for per-test
                             # coverage purposes.
                             report['source_files'] = [
                                 sf for sf in report['source_files'] if self.is_covered(sf)]
 
                             # Get baseline coverage
                             baseline_coverage = {}
-                            if self.config.get('per_test_category') == "web-platform":
-                                baseline_coverage = baseline_tests_cov['.html']
+                            if suite in baseline_tests_suite_cov:
+                                baseline_coverage = baseline_tests_suite_cov[suite]
+                            elif self.config.get('per_test_category') == "web-platform":
+                                baseline_coverage = baseline_tests_ext_cov['.html']
                             else:
-                                for file_type in baseline_tests_cov:
+                                for file_type in baseline_tests_ext_cov:
                                     if not test.endswith(file_type):
                                         continue
-                                    baseline_coverage = baseline_tests_cov[file_type]
+                                    baseline_coverage = baseline_tests_ext_cov[file_type]
                                     break
 
                             if not baseline_coverage:
                                 # Default to the '.js' baseline as it is the largest
                                 self.info("Did not find a baseline test for: " + test)
-                                baseline_coverage = baseline_tests_cov['.js']
+                                baseline_coverage = baseline_tests_ext_cov['.js']
 
                             unique_coverage = rm_baseline_cov(baseline_coverage, report)
 
                         with open(grcov_file, 'w') as f:
                             json.dump({
                                 'test': test,
                                 'suite': suite,
                                 'report': unique_coverage,