Bug 1418368 - Add a metrics for hero elementtiming - r?rwood draft
authorTarek Ziadé <tarek@mozilla.com>
Fri, 08 Dec 2017 09:35:49 +0100 (2017-12-08)
changeset 710489 a2ce183b0336a981e98f0d7a72791f98bac6e19d
parent 709618 437bfd403b76542b4b2e22430972b82da2d1cbbd
child 743590 dcdc38ce7d3fa1daca4abf9497252daf51fbe251
push id92834
push usertziade@mozilla.com
push dateSun, 10 Dec 2017 18:35:21 +0000 (2017-12-10)
reviewersrwood
bugs1418368
milestone59.0a1
Bug 1418368 - Add a metrics for hero elementtiming - r?rwood MozReview-Commit-ID: 1oGZ6YSjdRa
testing/talos/talos/cmdline.py
testing/talos/talos/config.py
testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
testing/talos/talos/pageloader/chrome/lh_fnbpaint.js
testing/talos/talos/pageloader/chrome/lh_hero.js
testing/talos/talos/pageloader/chrome/pageloader.js
testing/talos/talos/run_tests.py
testing/talos/talos/test.py
--- a/testing/talos/talos/cmdline.py
+++ b/testing/talos/talos/cmdline.py
@@ -72,16 +72,18 @@ def create_parser(mach_interface=False):
             help="Name of the subtest(s) to run (works only on DAMP)")
     add_arg('--mainthread', action='store_true',
             help="Collect mainthread IO data from the browser by setting"
                  " an environment variable")
     add_arg("--mozAfterPaint", action='store_true', dest="tpmozafterpaint",
             help="wait for MozAfterPaint event before recording the time")
     add_arg("--firstPaint", action='store_true', dest="firstpaint",
             help="Also report the first paint value in supported tests")
+    add_arg("--useHero", action='store_true', dest="tphero",
+            help="use Hero elementtiming attribute to record the time")
     add_arg("--userReady", action='store_true', dest="userready",
             help="Also report the user ready value in supported tests")
     add_arg('--spsProfile', action="store_true", dest="gecko_profile",
             help="(Deprecated - Use --geckoProfile instead.) Profile the "
                  "run and output the results in $MOZ_UPLOAD_DIR.")
     add_arg('--spsProfileInterval', dest='gecko_profile_interval', type=float,
             help="(Deprecated - Use --geckoProfileInterval instead.) How "
                  "frequently to take samples (ms)")
--- a/testing/talos/talos/config.py
+++ b/testing/talos/talos/config.py
@@ -35,16 +35,17 @@ DEFAULTS = dict(
         gecko_profile_entries=100000,
         resolution=1,
         mainthread=False,
         shutdown=False,
         timeout=3600,
         tpchrome=True,
         tpcycles=10,
         tpmozafterpaint=False,
+        tphero=False,
         fnbpaint=False,
         firstpaint=False,
         format_pagename=True,
         userready=False,
         testeventmap=[],
         base_vs_ref=False,
         tppagecycles=1,
         tploadnocache=False,
@@ -214,16 +215,17 @@ GLOBAL_OVERRIDES = (
     'gecko_profile',
     'gecko_profile_interval',
     'gecko_profile_entries',
     'tpcycles',
     'tppagecycles',
     'tpmanifest',
     'tptimeout',
     'tpmozafterpaint',
+    'tphero',
     'fnbpaint',
     'firstpaint',
     'userready',
 )
 
 
 CONF_VALIDATORS = []
 
@@ -347,32 +349,35 @@ def build_manifest(config, manifestName)
     newManifestName = manifestName + '.develop'
 
     # return new manifest
     return newManifestName
 
 
 def get_test(config, global_overrides, counters, test_instance):
     mozAfterPaint = getattr(test_instance, 'tpmozafterpaint', None)
+    hero = getattr(test_instance, 'tphero', None)
     firstPaint = getattr(test_instance, 'firstpaint', None)
     userReady = getattr(test_instance, 'userready', None)
     firstNonBlankPaint = getattr(test_instance, 'fnbpaint', None)
 
     test_instance.update(**global_overrides)
 
     # update original value of mozAfterPaint, this could be 'false',
     # so check for None
     if mozAfterPaint is not None:
         test_instance.tpmozafterpaint = mozAfterPaint
     if firstNonBlankPaint is not None:
         test_instance.fnbpaint = firstNonBlankPaint
     if firstPaint is not None:
         test_instance.firstpaint = firstPaint
     if userReady is not None:
         test_instance.userready = userReady
+    if hero is not None:
+        test_instance.tphero = hero
 
     # fix up url
     url = getattr(test_instance, 'url', None)
     if url:
         test_instance.url = utils.interpolate(convert_url(config, url))
 
     # fix up tpmanifest
     tpmanifest = getattr(test_instance, 'tpmanifest', None)
--- a/testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
+++ b/testing/talos/talos/mitmproxy/mitmproxy-playback-set.manifest
@@ -1,9 +1,9 @@
 [
     {
         "filename": "mitmproxy-recording-set-win10.zip",
-        "size": 9181715,
-        "digest": "61a4b32fc0a6b8408dce38872d8ac1a6a0e46622ef571c20d3e15b5f91709765268885b6a21126e79e75289c862312e22a91d8000ef6c7ada594bf677229574d",
+        "size": 9189938,
+        "digest": "e904917ed6bf1cef7201284385dc603a283e8e22f992876f17edcf0f1f20db95b609f0d8c7f593b4a0a6c20957dcb6a4d502c562ed74fb6cf4bc255c2f691f32",
         "algorithm": "sha512",
         "unpack": false
     }
 ]
\ No newline at end of file
--- a/testing/talos/talos/pageloader/chrome/lh_fnbpaint.js
+++ b/testing/talos/talos/pageloader/chrome/lh_fnbpaint.js
@@ -1,26 +1,28 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 var gRetryCounter = 0;
+var gErr = "Abort: firstNonBlankPaint value is not available after loading the page";
 
 
 function _contentFNBPaintHandler() {
   var x = content.window.performance.timing.timeToNonBlankPaint;
   if (typeof x == "undefined") {
-    sendAsyncMessage("PageLoader:FNBPaintError", {});
+    sendAsyncMessage("PageLoader:Error", {"msg": gErr});
   }
   if (x > 0) {
-    sendAsyncMessage("PageLoader:LoadEvent", {"fnbpaint": x});
+    sendAsyncMessage("PageLoader:LoadEvent", {"time": x,
+                                              "name": "fnbpaint"});
   } else {
     gRetryCounter += 1;
     if (gRetryCounter <= 10) {
       dump("fnbpaint is not yet available (0), retry number " + gRetryCounter + "...\n");
       content.setTimeout(_contentFNBPaintHandler, 100);
     } else {
       dump("unable to get a value for fnbpaint after " + gRetryCounter + " retries\n");
-      sendAsyncMessage("PageLoader:FNBPaintError", {});
+      sendAsyncMessage("PageLoader:Error", {"msg": gErr});
     }
   }
 }
 
 addEventListener("load", contentLoadHandlerCallback(_contentFNBPaintHandler), true); // eslint-disable-line no-undef
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/pageloader/chrome/lh_hero.js
@@ -0,0 +1,34 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+function _contentHeroHandler() {
+  var obs = null;
+  var el = content.window.document.querySelector("[elementtiming]");
+  if (el) {
+    function callback(entries, observer) {
+      entries.forEach(entry => {
+        sendAsyncMessage("PageLoader:LoadEvent",
+                         {"time": content.window.performance.now(),
+                          "name": "tphero"});
+        obs.disconnect();
+      });
+    }
+    // we want the element 100% visible on the viewport
+    var options = {root: null, rootMargin: "0px", threshold: [1]};
+    try {
+      obs = new content.window.IntersectionObserver(callback, options);
+      obs.observe(el);
+    } catch (err) {
+      sendAsyncMessage("PageLoader:Error", {"msg": err.message});
+    }
+  } else {
+      var err = "Could not find a tag with an elmenttiming attr on the page";
+      sendAsyncMessage("PageLoader:Error", {"msg": err});
+  }
+  return obs;
+}
+
+
+addEventListener("load", contentLoadHandlerCallback(_contentHeroHandler), true); // eslint-disable-line no-undef
--- a/testing/talos/talos/pageloader/chrome/pageloader.js
+++ b/testing/talos/talos/pageloader/chrome/pageloader.js
@@ -33,16 +33,17 @@ var pageCycle;
 var report;
 var timeout = -1;
 var delay = 250;
 var running = false;
 var forceCC = true;
 
 var useMozAfterPaint = false;
 var useFNBPaint = false;
+var useHero = false;
 var gPaintWindow = window;
 var gPaintListener = false;
 var loadNoCache = false;
 var scrollTest = false;
 var profilingInfo = false;
 var baseVsRef = false;
 var useBrowserChrome = false;
 
@@ -131,16 +132,17 @@ function plInit() {
       dumpLine("tp abort: talos.tpmanifest browser pref is not set");
       plStop(true);
     }
 
     NUM_CYCLES = Services.prefs.getIntPref("talos.tpcycles", 1);
     numPageCycles = Services.prefs.getIntPref("talos.tppagecycles", 1);
     timeout = Services.prefs.getIntPref("talos.tptimeout", -1);
     useMozAfterPaint = Services.prefs.getBoolPref("talos.tpmozafterpaint", false);
+    useHero = Services.prefs.getBoolPref("talos.tphero", false);
     useFNBPaint = Services.prefs.getBoolPref("talos.fnbpaint", false);
     loadNoCache = Services.prefs.getBoolPref("talos.tploadnocache", false);
     scrollTest = Services.prefs.getBoolPref("talos.tpscrolltest", false);
     useBrowserChrome = Services.prefs.getBoolPref("talos.tpchrome", false);
 
     // for pageloader tests the profiling info is found in an env variable
     // because it is not available early enough to set it as a browser pref
     var env = Components.classes["@mozilla.org/process/environment;1"].
@@ -233,16 +235,18 @@ function plInit() {
         content = browserWindow.getBrowser();
         content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/utils.js", false, true);
 
         // pick the right load handler
         if (useFNBPaint) {
           content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/lh_fnbpaint.js", false, true);
         } else if (useMozAfterPaint) {
           content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/lh_moz.js", false, true);
+        } else if (useHero) {
+          content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/lh_hero.js", false, true);
         } else {
           content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/lh_dummy.js", false, true);
 
         }
         content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/talos-content.js", false);
         content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/tscroll.js", false, true);
         content.selectedBrowser.messageManager.loadFrameScript("chrome://pageloader/content/Profiler.js", false, true);
 
@@ -260,17 +264,17 @@ function plInit() {
 function plPageFlags() {
   return pages[pageIndex].flags;
 }
 
 var ContentListener = {
   receiveMessage(message) {
     switch (message.name) {
       case "PageLoader:LoadEvent": return plLoadHandlerMessage(message);
-      case "PageLoader:FNBPaintError": return plFNBPaintErrorMessage(message);
+      case "PageLoader:Error": return plErrorMessage(message);
       case "PageLoader:RecordTime": return plRecordTimeMessage(message);
       case "PageLoader:IdleCallbackSet": return plIdleCallbackSet();
       case "PageLoader:IdleCallbackReceived": return plIdleCallbackReceived();
     }
     return undefined;
   },
 };
 
@@ -295,28 +299,24 @@ function plLoadPage() {
   }
 
   // messages to watch for page load
   let mm = content.selectedBrowser.messageManager;
   mm.addMessageListener("PageLoader:LoadEvent", ContentListener);
   mm.addMessageListener("PageLoader:RecordTime", ContentListener);
   mm.addMessageListener("PageLoader:IdleCallbackSet", ContentListener);
   mm.addMessageListener("PageLoader:IdleCallbackReceived", ContentListener);
-  if (useFNBPaint) {
-    mm.addMessageListener("PageLoader:FNBPaintError", ContentListener);
-  }
+  mm.addMessageListener("PageLoader:Error", ContentListener);
 
   removeLastAddedMsgListener = function() {
     mm.removeMessageListener("PageLoader:LoadEvent", ContentListener);
     mm.removeMessageListener("PageLoader:RecordTime", ContentListener);
     mm.removeMessageListener("PageLoader:IdleCallbackSet", ContentListener);
     mm.removeMessageListener("PageLoader:IdleCallbackReceived", ContentListener);
-    if (useFNBPaint) {
-      mm.removeMessageListener("PageLoader:FNBPaintError", ContentListener);
-    }
+    mm.removeMessageListener("PageLoader:Error", ContentListener);
   };
   failTimeout.register(loadFail, timeout);
   // record which page we are about to open
   TalosParentProfiler.mark("Opening " + pages[pageIndex].url.pathQueryRef);
 
   startAndLoadURI(pageName);
 }
 
@@ -607,23 +607,23 @@ function waitForPainted() {
 }
 
 function plPainted() {
   gPaintWindow.removeEventListener("MozAfterPaint", plPainted, true);
   gPaintListener = false;
   _loadHandler();
 }
 
-function _loadHandler(fnbpaint = 0) {
+function _loadHandler(paint_time = 0) {
   failTimeout.clear();
   var end_time = 0;
 
-  if (fnbpaint !== 0) {
+  if (paint_time !== 0) {
     // window.performance.timing.timeToNonBlankPaint is a timestamp
-    end_time = fnbpaint;
+    end_time = paint_time;
   } else {
     end_time = Date.now();
   }
 
   var duration = (end_time - start_time);
   TalosParentProfiler.pause("Bubbling load handler fired.");
 
   // does this page want to do its own timing?
@@ -634,19 +634,22 @@ function _loadHandler(fnbpaint = 0) {
   }
 
   plRecordTime(duration);
   plNextPage();
 }
 
 // the core handler
 function plLoadHandlerMessage(message) {
-  let fnbpaint = 0;
-  if (message.json.fnbpaint !== undefined) {
-    fnbpaint = message.json.fnbpaint;
+  let paint_time = 0;
+  // XXX message.json.name contains the name
+  // of the load handler, so in future versions
+  // we can record several times per load.
+  if (message.json.time !== undefined) {
+      paint_time = message.json.time;
   }
   failTimeout.clear();
 
   if ((plPageFlags() & EXECUTE_SCROLL_TEST)) {
     // Let the page settle down after its load event, then execute the scroll test.
     setTimeout(sendScroll, 500);
   } else if ((plPageFlags() & TEST_DOES_OWN_TIMING)) {
     var time;
@@ -663,38 +666,42 @@ function plLoadHandlerMessage(message) {
       gTime = -1;
     }
 
     if (time !== undefined) {
       plRecordTime(time);
       plNextPage();
     }
   } else {
-    _loadHandler(fnbpaint);
+    _loadHandler(paint_time);
   }
 }
 
 // the record time handler
 function plRecordTimeMessage(message) {
   gTime = message.json.time;
   gStartTime = message.json.startTime;
   recordedName = message.json.testName;
 
   if (useMozAfterPaint) {
     gStartTime = message.json.startTime;
   }
   _loadHandlerCapturing();
 }
 
-// error retrieving fnbpaint
-function plFNBPaintErrorMessage(message) {
-  dumpLine("Abort: firstNonBlankPaint value is not available after loading the page");
+
+// error
+function plErrorMessage(message) {
+  if (message.json.msg) {
+    dumpLine(message.json.msg);
+  }
   plStop(true);
 }
 
+
 function plStop(force) {
   plStopAll(force);
 }
 
 function plStopAll(force) {
   try {
     if (force == false) {
       pageIndex = 0;
@@ -721,26 +728,22 @@ function plStopAll(force) {
     if (useMozAfterPaint) {
       content.removeEventListener("MozAfterPaint", plPaintedCapturing, true);
       content.removeEventListener("MozAfterPaint", plPainted, true);
     }
 
     let mm = content.selectedBrowser.messageManager;
     mm.removeMessageListener("PageLoader:LoadEvent", ContentListener);
     mm.removeMessageListener("PageLoader:RecordTime", ContentListener);
-
-    if (useFNBPaint) {
-      mm.removeMessageListener("PageLoader:FNBPaintError", ContentListener);
-    }
+    mm.removeMessageListener("PageLoader:Error", ContentListener);
 
     if (isIdleCallbackPending) {
       mm.removeMessageListener("PageLoader:IdleCallbackSet", ContentListener);
       mm.removeMessageListener("PageLoader:IdleCallbackReceived", ContentListener);
     }
-
     mm.loadFrameScript("data:,removeEventListener('load', _contentLoadHandler, true);", false, true);
   }
 
   if (MozillaFileLogger && MozillaFileLogger._foStream)
     MozillaFileLogger.close();
 
   goQuitApplication();
 }
--- a/testing/talos/talos/run_tests.py
+++ b/testing/talos/talos/run_tests.py
@@ -48,17 +48,18 @@ def set_tp_preferences(test, browser_con
     # if profiling is on, override tppagecycles to prevent test hanging
     if test['gecko_profile']:
         LOG.info("Gecko profiling is enabled so talos is reducing the number "
                  "of cycles, please disregard reported numbers")
         for cycle_var in ['tppagecycles', 'tpcycles', 'cycles']:
             if test[cycle_var] > 2:
                 test[cycle_var] = 2
 
-    CLI_bool_options = ['tpchrome', 'tpmozafterpaint', 'tploadnocache', 'tpscrolltest', 'fnbpaint']
+    CLI_bool_options = ['tpchrome', 'tphero', 'tpmozafterpaint', 'tploadnocache', 'tpscrolltest',
+                        'fnbpaint']
     CLI_options = ['tpcycles', 'tppagecycles', 'tptimeout', 'tpmanifest']
     for key in CLI_bool_options:
         if key in test:
             _pref_name = "talos.%s" % key
             test['preferences'][_pref_name] = test.get(key)
 
     for key in CLI_options:
         value = test.get(key)
--- a/testing/talos/talos/test.py
+++ b/testing/talos/talos/test.py
@@ -100,16 +100,17 @@ class TsBase(Test):
         'gecko_profile_startup',
         'preferences',
         'xperf_counters',
         'xperf_providers',
         'xperf_user_providers',
         'xperf_stackwalk',
         'tpmozafterpaint',
         'fnbpaint',
+        'tphero',
         'profile',
         'firstpaint',
         'userready',
         'testeventmap',
         'base_vs_ref',
         'extensions',
         'filters',
         'setup',
@@ -235,19 +236,19 @@ class tresize(TsBase):
 
 class PageloaderTest(Test):
     """abstract base class for a Talos Pageloader test"""
     tpmanifest = None  # test manifest
     tpcycles = 1  # number of time to run each page
     cycles = None
     timeout = None
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
-            'tpmozafterpaint', 'fnbpaint', 'tploadnocache', 'firstpaint', 'userready',
-            'testeventmap', 'base_vs_ref', 'mainthread', 'resolution', 'cycles',
-            'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
+            'tpmozafterpaint', 'fnbpaint', 'tphero', 'tploadnocache', 'firstpaint',
+            'userready', 'testeventmap', 'base_vs_ref', 'mainthread', 'resolution',
+            'cycles', 'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
             'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
             'tpscrolltest', 'xperf_counters', 'timeout', 'responsiveness',
             'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
             'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile']
 
 
 class QuantumPageloadTest(PageloaderTest):
@@ -879,16 +880,17 @@ class perf_reftest_singletons(Pageloader
 
 
 @register_test()
 class tp6_google(QuantumPageloadTest):
     """
     Quantum Pageload Test - Google
     """
     tpmanifest = '${talos}/tests/quantum_pageload/quantum_pageload_google.manifest'
+    tphero = True
 
 
 @register_test()
 class tp6_google_heavy(tp6_google):
     """
     tp6_google test ran against a heavy-user profile
     """
     profile = 'simple'