author Rob Wood <rwood@mozilla.com>

Thu, 21 Sep 2017 08:47:56 -0400

changeset 668280 cbcf0864465448607cf2d99c8312be4a5902dc7a

parent 668025 47f7b6c64265bc7bdd22eef7ab71abc97cf3f8bf

child 732658 49083820ac8b8d0a046d4cc98d0e7801fb1c9097

push id 80999

push user rwood@mozilla.com

push date Thu, 21 Sep 2017 12:50:49 +0000

reviewers jmaher

bugs 1397438

milestone 57.0a1

testing/talos/talos.json file | annotate | diff | comparison | revisions

testing/talos/talos/output.py file | annotate | diff | comparison | revisions

testing/talos/talos/pageloader/chrome/pageloader.js file | annotate | diff | comparison | revisions

testing/talos/talos/pageloader/install.rdf file | annotate | diff | comparison | revisions

testing/talos/talos/pageloader/pageloader-signed.xpi file | annotate | diff | comparison | revisions

testing/talos/talos/results.py file | annotate | diff | comparison | revisions

testing/talos/talos/run_tests.py file | annotate | diff | comparison | revisions

testing/talos/talos/test.py file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/.eslintrc.json file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/bloom-basic-2.html file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/bloom-basic-ref.html file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/bloom-basic.html file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/bloom_basic.manifest file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/bloom_basic_ref.manifest file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/coalesce-1.html file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/coalesce-ref.html file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/perf_reftest.manifest file | annotate | diff | comparison | revisions

testing/talos/talos/tests/perf-reftest/util.js file | annotate | diff | comparison | revisions
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -65,20 +65,20 @@
         "svgr-e10s": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "tsvg_static"]
         },
         "svgr-stylo-disabled-e10s": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "tsvg_static"],
             "talos_options": ["--disable-stylo"]
         },
         "perf-reftest-e10s": {
-            "tests": ["bloom_basic"]
+            "tests": ["perf_reftest"]
         },
         "perf-reftest-stylo-disabled-e10s": {
-            "tests": ["bloom_basic"],
+            "tests": ["perf_reftest"],
             "talos_options": ["--disable-stylo"]
         },
         "perf-reftest-singletons-e10s": {
             "tests": ["perf_reftest_singletons"]
         },
         "perf-reftest-singletons-stylo-disabled-e10s": {
             "tests": ["perf_reftest_singletons"],
             "talos_options": ["--disable-stylo"]
--- a/testing/talos/talos/output.py
+++ b/testing/talos/talos/output.py
@@ -87,34 +87,38 @@ class Output(object):
                     # override the list of page results for each run
                     result.results = results
 
                 for result in tresults:
                     filtered_results = \
                         result.values(suite['name'],
                                       test.test_config['filters'])
                     vals.extend([[i['value'], j] for i, j in filtered_results])
+                    subtest_index = 0
                     for val, page in filtered_results:
                         if page == 'NULL':
                             # no real subtests
                             page = test.name()
                         subtest = {
                             'name': page,
                             'value': val['filtered'],
                             'replicates': replicates[page],
                         }
                         # if results are from a comparison test i.e. perf-reftest, it will also
                         # contain replicates for 'base' and 'reference'; we wish to keep those
                         # to reference; actual results were calculated as the difference of those
-                        base_runs = result.results[0].get('base_runs', None)
-                        ref_runs = result.results[0].get('ref_runs', None)
+                        base_runs = result.results[subtest_index].get('base_runs', None)
+                        ref_runs = result.results[subtest_index].get('ref_runs', None)
                         if base_runs and ref_runs:
                             subtest['base_replicates'] = base_runs
                             subtest['ref_replicates'] = ref_runs
+
                         subtests.append(subtest)
+                        subtest_index += 1
+
                         if test.test_config.get('lower_is_better') is not None:
                             subtest['lowerIsBetter'] = \
                                 test.test_config['lower_is_better']
                         if test.test_config.get('alert_threshold') is not None:
                             subtest['alertThreshold'] = \
                                 test.test_config['alert_threshold']
                         if test.test_config.get('unit'):
                             subtest['unit'] = test.test_config['unit']
--- a/testing/talos/talos/pageloader/chrome/pageloader.js
+++ b/testing/talos/talos/pageloader/chrome/pageloader.js
@@ -46,16 +46,17 @@ var useMozAfterPaint = false;
 var useFNBPaint = false;
 var gPaintWindow = window;
 var gPaintListener = false;
 var loadNoCache = false;
 var scrollTest = false;
 var gDisableE10S = false;
 var gUseE10S = false;
 var profilingInfo = false;
+var baseVsRef = false;
 
 var isIdleCallbackPending = false;
 
 // when TEST_DOES_OWN_TIMING, we need to store the time from the page as MozAfterPaint can be slower than pageload
 var gTime = -1;
 var gStartTime = -1;
 var gReference = -1;
 
@@ -197,17 +198,16 @@ function plInit() {
     if (startIndex > endIndex) {
       dumpLine("tp: error: startIndex >= endIndex");
       plStop(true);
     }
 
     pages = pages.slice(startIndex, endIndex + 1);
     pageUrls = pages.map(function(p) { return p.url.spec.toString(); });
     report = new Report();
-
     if (doRenderTest)
       renderReport = new Report();
 
     pageIndex = 0;
     if (profilingInfo) {
       TalosParentProfiler.beginTest(getCurrentPageShortName());
     }
 
@@ -395,19 +395,17 @@ function plLoadPage() {
     mm.removeMessageListener("PageLoader:LoadEvent", ContentListener);
     mm.removeMessageListener("PageLoader:RecordTime", ContentListener);
     mm.removeMessageListener("PageLoader:IdleCallbackSet", ContentListener);
     mm.removeMessageListener("PageLoader:IdleCallbackReceived", ContentListener);
     if (useFNBPaint) {
       mm.removeMessageListener("PageLoader:FNBPaintError", ContentListener);
     }
   };
-
   failTimeout.register(loadFail, timeout);
-
   // record which page we are about to open
   TalosParentProfiler.mark("Opening " + pages[pageIndex].url.pathQueryRef);
 
   if (reportRSS) {
     collectMemory(startAndLoadURI, pageName);
   } else {
     startAndLoadURI(pageName);
   }
@@ -560,17 +558,25 @@ function plRecordTime(time) {
   var i = pageIndex
   if (i < pages.length - 1) {
     i++;
   } else {
     i = 0;
   }
   var nextName = pages[i].url.spec;
   if (!recordedName) {
-    recordedName = pageUrls[pageIndex];
+    // when doing base vs ref type of test, add pre 'base' or 'ref' to reported page name;
+    // this is necessary so that if multiple subtests use same reference page, results for
+    // each ref page run will be kept separate for each base vs ref run, and not grouped
+    // into just one set of results values for everytime that reference page was loaded
+    if (baseVsRef) {
+      recordedName = pages[pageIndex].pre + pageUrls[pageIndex];
+    } else {
+      recordedName = pageUrls[pageIndex];
+    }
   }
   if (typeof(time) == "string") {
     var times = time.split(",");
     var names = recordedName.split(",");
     for (var t = 0; t < times.length; t++) {
       if (names.length == 1) {
         report.recordTime(names, times[t]);
       } else {
@@ -888,20 +894,21 @@ function plLoadURLsFromURI(manifestUri) 
     fstream.init(uriFile.file, -1, 0, 0);
   } catch (ex) {
       dumpLine("tp: the file %s doesn't exist" % uriFile.file);
       return null;
   }
 
   var lstream = fstream.QueryInterface(Ci.nsILineInputStream);
 
-  var d = [];
+  var url_array = [];
 
   var lineNo = 0;
   var line = {value: null};
+  var baseVsRefIndex = 0;
   var more;
   do {
     lineNo++;
     more = lstream.readLine(line);
     var s = line.value;
 
     // strip comments (only leading ones)
     s = s.replace(/^#.*/, "");
@@ -909,16 +916,17 @@ function plLoadURLsFromURI(manifestUri) 
     // strip leading and trailing whitespace
     s = s.replace(/^\s*/, "").replace(/\s*$/, "");
 
     if (!s)
       continue;
 
     var flags = 0;
     var urlspec = s;
+    baseVsRefIndex += 1;
 
     // split on whitespace, and figure out if we have any flags
     var items = s.split(/\s+/);
     if (items[0] == "include") {
       if (items.length != 2) {
         dumpLine("tp: Error on line " + lineNo + " in " + manifestUri.spec + ": include must be followed by the manifest to include!");
         return null;
       }
@@ -927,17 +935,17 @@ function plLoadURLsFromURI(manifestUri) 
       if (subManifest == null) {
         dumpLine("tp: invalid URI on line " + manifestUri.spec + ":" + lineNo + " : '" + line.value + "'");
         return null;
       }
 
       var subItems = plLoadURLsFromURI(subManifest);
       if (subItems == null)
         return null;
-      d = d.concat(subItems);
+      url_array = url_array.concat(subItems);
     } else {
       // For scrollTest flag, we accept "normal" pages but treat them as TEST_DOES_OWN_TIMING
       // together with EXECUTE_SCROLL_TEST which makes us run the scroll test on load.
       // We do this by artificially "injecting" the TEST_DOES_OWN_TIMING flag ("%") to the item
       // and then let the default flow for this flag run without further modifications
       // (other than calling the scroll test once the page is loaded).
       // Note that if we have the scrollTest flag but the item already has "%", then we do
       // nothing (the scroll test will not execute, and the page will report with its
@@ -947,32 +955,68 @@ function plLoadURLsFromURI(manifestUri) 
         flags |= EXECUTE_SCROLL_TEST;
       }
 
       if (items.length == 2) {
         if (items[0].indexOf("%") != -1)
           flags |= TEST_DOES_OWN_TIMING;
 
         urlspec = items[1];
+      } else if (items.length == 3) {
+        // base vs ref type of talos test
+        // expect each manifest line to be in the format of:
+        // & http://localhost/tests/perf-reftest/base-page.html, http://localhost/tests/perf-reftest/reference-page.html
+        // test will run with the base page, then with the reference page; and ultimately the actual test results will
+        // be the comparison values of those two pages; more than one line will result in base vs ref subtests
+        if (items[0].indexOf("&") != -1) {
+          baseVsRef = true;
+          flags |= TEST_DOES_OWN_TIMING;
+          // for the base, must remove the comma on the end of the actual url
+          var urlspecBase = items[1].slice(0, -1);
+          var urlspecRef = items[2];
+        } else {
+          dumpLine("tp: Error on line " + lineNo + " in " + manifestUri.spec + ": unknown manifest format!");
+          return null;
+        }
       } else if (items.length != 1) {
         dumpLine("tp: Error on line " + lineNo + " in " + manifestUri.spec + ": whitespace must be %-escaped!");
         return null;
       }
 
-      var url = gIOS.newURI(urlspec, null, manifestUri);
+      var url;
+
+      if (!baseVsRef) {
+        url = gIOS.newURI(urlspec, null, manifestUri);
+
+        if (pageFilterRegexp && !pageFilterRegexp.test(url.spec))
+          continue;
 
-      if (pageFilterRegexp && !pageFilterRegexp.test(url.spec))
-        continue;
+        url_array.push({ url, flags });
+      } else {
+        // base vs ref type of talos test
+        // we add a 'pre' prefix here indicating that this particular page is a base page or a reference
+        // page; later on this 'pre' is used when recording the actual time value/result; because in
+        // the results we use the url as the results key; but we might use the same test page as a reference
+        // page in the same test suite, so we need to add a prefix so this results key is always unique
+        url = gIOS.newURI(urlspecBase, null, manifestUri);
+        if (pageFilterRegexp && !pageFilterRegexp.test(url.spec))
+          continue;
+        var pre = "base_page_" + baseVsRefIndex + "_";
+        url_array.push({ url, flags, pre });
 
-      d.push({   url,
-               flags });
+        url = gIOS.newURI(urlspecRef, null, manifestUri);
+        if (pageFilterRegexp && !pageFilterRegexp.test(url.spec))
+          continue;
+        pre = "ref_page_" + baseVsRefIndex + "_";
+        url_array.push({ url, flags, pre });
+      }
     }
   } while (more);
 
-  return d;
+  return url_array;
 }
 
 function dumpLine(str) {
   if (MozillaFileLogger && MozillaFileLogger._foStream)
     MozillaFileLogger.log(str + "\n");
   dump(str);
   dump("\n");
 }
--- a/testing/talos/talos/pageloader/install.rdf
+++ b/testing/talos/talos/pageloader/install.rdf
@@ -1,15 +1,15 @@
 <?xml version="1.0"?>
 
 <RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
      xmlns:em="http://www.mozilla.org/2004/em-rdf#">
   <Description about="urn:mozilla:install-manifest">
     <em:id>pageloader@mozilla.org</em:id>
-    <em:version>1.0.27</em:version>
+    <em:version>1.0.29</em:version>
     <em:targetApplication>
       <Description>
         <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
         <em:minVersion>44.0</em:minVersion>
         <em:maxVersion>*</em:maxVersion>
       </Description>
     </em:targetApplication>
     <!-- Front End MetaData -->
--- a/testing/talos/talos/results.py
+++ b/testing/talos/talos/results.py
@@ -248,17 +248,22 @@ class PageloaderResults(Results):
             self.results.append(result)
 
     def format_pagename(self, page):
         """
         fix up the page for reporting
         """
         page = page.rstrip('/')
         if '/' in page:
-            page = page.split('/')[0]
+            if 'base_page' in page or 'ref_page' in page:
+                # for base vs ref type test, the page name is different format, i.e.
+                # base_page_1_http://localhost:53309/tests/perf-reftest/bloom-basic.html
+                page = page.split('/')[-1]
+            else:
+                page = page.split('/')[0]
         return page
 
 
 class BrowserLogResults(object):
     """parse the results from the browser log output"""
 
     # tokens for the report types
     report_tokens = [
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -336,39 +336,55 @@ def make_comparison_result(base_and_refe
     results object with the comparison results, that will look like traditional single test results
     like this:
 
     PERFHERDER_DATA: {"framework": {"name": "talos"}, "suites": [{"lowerIsBetter": true,
     "subtests": [{"name": "", "lowerIsBetter": true, "alertThreshold": 5.0, "replicates":
     [16.705, ...], "value": 16.705, "unit": "ms"}], "extraOptions": ["e10s"], "name":
     "bloom_basic", "alertThreshold": 5.0}]}
     '''
-    # separate the 'base' and 'reference' result run values
-    base_result_runs = base_and_reference_results.results[0].results[0]['runs']
-    ref_result_runs = base_and_reference_results.results[0].results[1]['runs']
-
     # create a new results object for the comparison result; keep replicates from both pages
     comparison_result = copy.deepcopy(base_and_reference_results)
 
     # remove original results from our copy as they will be replaced by one comparison result
     comparison_result.results[0].results = []
+    comp_results = comparison_result.results[0].results
 
-    # populate our new comparison result with 'base' and 'ref' replicates
-    comparison_result.results[0].results.append({'index': 0,
-                                                 'runs': [],
-                                                 'page': '',
-                                                 'base_runs': base_result_runs,
-                                                 'ref_runs': ref_result_runs})
+    # zero-based count of how many base vs reftest sets we have
+    subtest_index = 0
+
+    # each set of two results is actually a base test followed by the
+    # reference test; we want to go through each set of base vs reference
+    for x in range(0, len(base_and_reference_results.results[0].results), 2):
+
+        # separate the 'base' and 'reference' result run values
+        results = base_and_reference_results.results[0].results
+        base_result_runs = results[x]['runs']
+        ref_result_runs = results[x + 1]['runs']
+
+        # the test/subtest result is the difference between the base vs reference test page
+        # values; for this result use the base test page name for the subtest/test name
+        sub_test_name = base_and_reference_results.results[0].results[x]['page']
 
-    # now step thru each result, compare 'base' vs 'ref', and store the difference in 'runs'
-    _index = 0
-    for next_ref in comparison_result.results[0].results[0]['ref_runs']:
-        diff = abs(next_ref - comparison_result.results[0].results[0]['base_runs'][_index])
-        comparison_result.results[0].results[0]['runs'].append(round(diff, 3))
-        _index += 1
+        # populate our new comparison result with 'base' and 'ref' replicates
+        comp_results.append({'index': 0,
+                             'runs': [],
+                             'page': sub_test_name,
+                             'base_runs': base_result_runs,
+                             'ref_runs': ref_result_runs})
+
+        # now step thru each result, compare 'base' vs 'ref', and store the difference in 'runs'
+        _index = 0
+        for next_ref in comp_results[subtest_index]['ref_runs']:
+            diff = abs(next_ref - comp_results[subtest_index]['base_runs'][_index])
+            comp_results[subtest_index]['runs'].append(round(diff, 3))
+            _index += 1
+
+        # increment our base vs reference subtest index
+        subtest_index += 1
 
     return comparison_result
 
 
 def convert_to_separate_test_results(multi_value_result, test_event_map):
     ''' Receive a test result that actually contains multiple values in a single iteration, and
     parse it out in order to 'fake' three seprate test results.
 
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -802,24 +802,24 @@ class a11yr(PageloaderTest):
     tpmozafterpaint = True
     tpchrome = False
     preferences = {'dom.send_after_paint_to_content': False}
     unit = 'ms'
     alert_threshold = 5.0
 
 
 @register_test()
-class bloom_basic(PageloaderTest):
+class perf_reftest(PageloaderTest):
     """
-    Stylo bloom_basic: runs bloom_basic and bloom_basic_ref and reports difference
+    Style perf-reftest a set of tests where the result is the difference of base vs ref pages
     """
     base_vs_ref = True  # compare the two test pages with eachother and report comparison
-    tpmanifest = '${talos}/tests/perf-reftest/bloom_basic.manifest'
+    tpmanifest = '${talos}/tests/perf-reftest/perf_reftest.manifest'
     tpcycles = 1
-    tppagecycles = 25
+    tppagecycles = 10
     gecko_profile_interval = 1
     gecko_profile_entries = 2000000
     filters = filter.ignore_first.prepare(5) + filter.median.prepare()
     unit = 'ms'
     lower_is_better = True
     alert_threshold = 5.0
 
 
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/perf-reftest/.eslintrc.json
@@ -0,0 +1,6 @@
+{
+    "rules": {
+        "no-undef": "off"
+    }
+}
+
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/perf-reftest/bloom-basic-2.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <script src="util.js"></script>
+    <script>
+window.onload = function() {
+  /* Use many rules, instead of one rule with many selectors, to test per-rule overhead. */
+  document.head.appendChild(build_rule("caption div, caption span", 1, "{ color: blue; } ", 20000));
+  let dom = build_dom(5000, "div");
+
+  flush_style();
+  perf_start();
+  document.body.appendChild(dom);
+  flush_style(dom);
+  perf_finish();
+}
+    </script>
+  </head>
+  <body>
+  </body>
+</html>
--- a/testing/talos/talos/tests/perf-reftest/bloom-basic-ref.html
+++ b/testing/talos/talos/tests/perf-reftest/bloom-basic-ref.html
@@ -1,34 +1,20 @@
 <!DOCTYPE html>
 <html>
   <head>
     <script src="util.js"></script>
-    <script src="chrome://talos-powers-content/content/TalosContentProfiler.js"></script>
     <script>
-/* import-globals-from util.js */
 window.onload = function() {
-  document.head.appendChild(build_rule("span > div", 10000, "{ color: blue; } "));
+  document.head.appendChild(build_rule("caption > div, caption > span", 20000, "{ color: blue; } "));
   let dom = build_dom(5000, "div");
 
-  TalosContentProfiler.resume("bloom_basic_ref start", true).then(() => {
-    let start = performance.now();
-    document.body.appendChild(dom);
-    window.foo = window.getComputedStyle(document.body).color;
-    let end = performance.now();
-
-    TalosContentProfiler.pause("bloom_basic_ref finish", true).then(() => {
-      // when running in talos report results; when running outside talos just alert
-      if (window.tpRecordTime) {
-        window.tpRecordTime(end - start, start);
-      } else {
-        console.log(end);
-        console.log(start);
-        alert("Result: " + (end - start).toFixed(2) + " (ms)");
-      }
-    });
-  });
+  flush_style();
+  perf_start();
+  document.body.appendChild(dom);
+  flush_style(dom);
+  perf_finish();
 }
     </script>
   </head>
   <body>
   </body>
 </html>
--- a/testing/talos/talos/tests/perf-reftest/bloom-basic.html
+++ b/testing/talos/talos/tests/perf-reftest/bloom-basic.html
@@ -1,34 +1,20 @@
 <!DOCTYPE html>
 <html>
   <head>
     <script src="util.js"></script>
-    <script src="chrome://talos-powers-content/content/TalosContentProfiler.js"></script>
     <script>
-/* import-globals-from util.js */
 window.onload = function() {
-  document.head.appendChild(build_rule("span div", 10000, "{ color: blue; } "));
+  document.head.appendChild(build_rule("caption div, caption span", 20000, "{ color: blue; } "));
   let dom = build_dom(5000, "div");
 
-  TalosContentProfiler.resume("bloom_basic start", true).then(() => {
-    let start = performance.now();
-    document.body.appendChild(dom);
-    window.foo = window.getComputedStyle(document.body).color;
-    let end = performance.now();
-
-    TalosContentProfiler.pause("bloom_basic end", true).then(() => {
-      // when running in talos report results; when running outside talos just alert
-      if (window.tpRecordTime) {
-        window.tpRecordTime(end - start, start);
-      } else {
-        console.log(end);
-        console.log(start);
-        alert("Result: " + (end - start).toFixed(2) + " (ms)");
-      }
-    });
-  });
+  flush_style();
+  perf_start();
+  document.body.appendChild(dom);
+  flush_style(dom);
+  perf_finish();
 }
     </script>
   </head>
   <body>
   </body>
 </html>
deleted file mode 100644
--- a/testing/talos/talos/tests/perf-reftest/bloom_basic.manifest
+++ /dev/null
@@ -1,4 +0,0 @@
-# base_vs_ref is set in test.py for this test, so each of these pages are run as separate
-# tests, but then compared against eachother; and the reported results are the comparison
-% http://localhost/tests/perf-reftest/bloom-basic.html
-% http://localhost/tests/perf-reftest/bloom-basic-ref.html
deleted file mode 100644
--- a/testing/talos/talos/tests/perf-reftest/bloom_basic_ref.manifest
+++ /dev/null
@@ -1,1 +0,0 @@
-% http://localhost/tests/perf-reftest/bloom-basic-ref.html
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/perf-reftest/coalesce-1.html
@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<style>
+.x { color: yellow; }
+</style>
+<script src="util.js"></script>
+<script>
+function set_class(n) {
+  n.className = "x";
+  n = n.firstChild;
+  while (n) {
+    set_class(n);
+    n = n.nextSibling;
+  }
+}
+window.onload = function() {
+  let root = build_dom(50000, "div");
+  document.body.appendChild(root);
+  flush_style(root);
+  perf_start();
+  set_class(root);
+  flush_style(root);
+  perf_finish();
+};
+</script>
+<body></body>
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/perf-reftest/coalesce-ref.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<style>
+.x, .x * { color: yellow; }
+</style>
+<script src="util.js"></script>
+<script>
+window.onload = function() {
+  let root = build_dom(50000, "div");
+  document.body.appendChild(root);
+  flush_style(root);
+  root.className = "x";
+  perf_start();
+  flush_style(root);
+  perf_finish();
+};
+</script>
+<body></body>
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/tests/perf-reftest/perf_reftest.manifest
@@ -0,0 +1,6 @@
+# base_vs_ref is set in test.py for this test, so each of these lines is the base vs ref test pag
+# both pages are run and then the results compared, the ultimate reported results are the comparisons.
+# Each line is reported as a subtest like the other talos tests that have multiple lines of URLs.
+& http://localhost/tests/perf-reftest/bloom-basic.html, http://localhost/tests/perf-reftest/bloom-basic-ref.html
+& http://localhost/tests/perf-reftest/bloom-basic-2.html, http://localhost/tests/perf-reftest/bloom-basic-ref.html
+& http://localhost/tests/perf-reftest/coalesce-1.html, http://localhost/tests/perf-reftest/coalesce-ref.html
--- a/testing/talos/talos/tests/perf-reftest/util.js
+++ b/testing/talos/talos/tests/perf-reftest/util.js
@@ -1,17 +1,82 @@
-function build_dom(n, elemName) {
+var perf_data = {
+  start: null,
+  end: null,
+}
+
+function build_dom(n, elemName, options) {
+  // By default we use different elements in the DOM to defeat the style sharing
+  // cache, otherwise this sythetic DOM is trivially stylable by engines with that
+  // optimization.
+  options = options || {};
+  var elemNameLeft = options.elemNameLeft || "div";
+  var elemNameRight = options.elemNameRight || "span";
+
   var ours = document.createElement(elemName);
+  for (var attr in options.attributes) {
+    ours.setAttribute(attr, options.attributes[attr]);
+  }
+
   if (n != 1) {
     var leftSize = Math.floor(n / 2);
     var rightSize = Math.floor((n - 1) / 2);
-    ours.appendChild(build_dom(leftSize, elemName));
+    ours.appendChild(build_dom(leftSize, elemNameLeft, options));
     if (rightSize > 0)
-      ours.appendChild(build_dom(rightSize, elemName));
+      ours.appendChild(build_dom(rightSize, elemNameRight, options));
   }
   return ours;
 }
 
-function build_rule(selector, selectorRepeat, declaration) {
+function build_rule(selector, selectorRepeat, declaration, ruleRepeat) {
+  ruleRepeat = ruleRepeat || 1;
   var s = document.createElement("style");
-  s.textContent = Array(selectorRepeat).fill(selector).join(", ") + declaration;
+  var rule = Array(selectorRepeat).fill(selector).join(", ") + declaration;
+  s.textContent = Array(ruleRepeat).fill(rule).join("\n\n");
   return s;
 }
+
+function flush_style(element) {
+  getComputedStyle(element || document.documentElement).color;
+}
+
+function perf_start() {
+  if (perf_data.start !== null) {
+    throw "already started timing!";
+  }
+
+  perf_data.start = performance.now();
+}
+
+function perf_finish() {
+  var end = performance.now();
+
+  if (perf_data.start === null) {
+    throw "haven't started timing!";
+  }
+
+  if (perf_data.end !== null) {
+    throw "already finished timing!";
+  }
+
+  var start = perf_data.start;
+  perf_data.end = end;
+
+  // when running in talos report results; when running outside talos just alert
+  if (window.tpRecordTime) {
+    // Running in talos.
+    window.tpRecordTime(end - start, start);
+  } else if (window.parent && window.parent.report_perf_reftest_time) {
+    // Running in the perf-reftest runner.
+    window.parent.report_perf_reftest_time({ type: "time", value: end - start });
+  } else {
+    // Running standalone; just alert.
+    console.log(end);
+    console.log(start);
+    alert("Result: " + (end - start).toFixed(2) + " (ms)");
+  }
+}
+
+if (window.parent.report_perf_reftest_time) {
+  window.addEventListener("error", function(e) {
+    window.parent.report_perf_reftest_time({ type: "error", value: e.message });
+  });
+}
author	Rob Wood <rwood@mozilla.com>
	Thu, 21 Sep 2017 08:47:56 -0400
changeset 668280	cbcf0864465448607cf2d99c8312be4a5902dc7a
parent 668025	47f7b6c64265bc7bdd22eef7ab71abc97cf3f8bf
child 732658	49083820ac8b8d0a046d4cc98d0e7801fb1c9097
push id	80999
push user	rwood@mozilla.com
push date	Thu, 21 Sep 2017 12:50:49 +0000
reviewers	jmaher
bugs	1397438
milestone	57.0a1
testing/talos/talos.json		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/output.py		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/pageloader/chrome/pageloader.js		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/pageloader/install.rdf		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/pageloader/pageloader-signed.xpi		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/results.py		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/run_tests.py		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/test.py		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/.eslintrc.json		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/bloom-basic-2.html		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/bloom-basic-ref.html		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/bloom-basic.html		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/bloom_basic.manifest		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/bloom_basic_ref.manifest		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/coalesce-1.html		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/coalesce-ref.html		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/perf_reftest.manifest		file \| annotate \| diff \| comparison \| revisions
testing/talos/talos/tests/perf-reftest/util.js		file \| annotate \| diff \| comparison \| revisions