Bug 1469280 - Fix raptor speedometer score calculation; r?jmaher draft
authorRob Wood <rwood@mozilla.com>
Wed, 20 Jun 2018 19:13:16 -0400
changeset 808841 166e7c3462c5e885d2e2cbf12839551585410055
parent 808764 7af2b112f3758a63801689d185eea7d03f3a1be0
push id113520
push userrwood@mozilla.com
push dateWed, 20 Jun 2018 23:19:04 +0000
reviewersjmaher
bugs1469280
milestone62.0a1
Bug 1469280 - Fix raptor speedometer score calculation; r?jmaher MozReview-Commit-ID: FxyanRA2LLk
testing/raptor/raptor/output.py
testing/raptor/raptor/tests/raptor-speedometer.ini
third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -81,37 +81,69 @@ class Output(object):
                     new_subtest['unit'] = test.unit
 
                     filtered_values = filter.ignore_first(new_subtest['replicates'], 1)
                     new_subtest['value'] = filter.median(filtered_values)
                     vals.append(new_subtest['value'])
 
                     subtests.append(new_subtest)
 
-            elif test.type == "benchmark":
-                # each benchmark 'index' becomes a subtest; each pagecycle / iteration
-                # of the test has multiple values per index/subtest
+            elif test.type == "benchmark" and 'speedometer' in test.name:
+
+                # results come from benchmark (via control server) formatted as an array,
+                # that contains results from each pagecycle. Each pagecycle is formatted
+                # as: array[0] contains a list of test values, in sets of 5 (speedometer
+                # has 5 values per single run per subsuite). array[1] contains a list of
+                # of corresponding subsuite names (subsuite name duplicated 5x)
+                # so for each single pagecycle, there are 160 sets of 5 vals (800 vals total)
+                # and there are 800 corresponding subsuite names (160 * 5)
+
+                # first lets convert the pagecycle to a list of dicts - where each dict
+                # item will be { 'subsuite_name': [val1, val2, val3, val4, val5] }
+                # so only one dict entry for each of the 160 unique subsuite names
+
+                overall_suite_results = []
 
-                # this is the format we receive the results in from the benchmark
-                # i.e. this is ONE pagecycle of speedometer:
+                for page_cycle in test.measurements['speedometer']:
+                    # use an OrderedDict so the order entries are added is persisted
+                    # we need the speedometer subsuites to be in the order they're reported
+                    from collections import OrderedDict
+                    page_cycle_results = OrderedDict()
+
+                    page_cycle_vals = page_cycle[0]
+                    page_cycle_names = page_cycle[1]
+
+                    for count in range(len(page_cycle_vals)):
+                        # page_cycle_vals[count] = [val1, val2, val3, val4, val5]
+                        # this covers/is for the corresponding 5 subsuite names
+                        new_entry = {}
+                        new_entry[page_cycle_names[count * 5]] = page_cycle_vals[count]
+                        page_cycle_results.update(new_entry)
+
+                    # now we have that pagecycle results; add to overall suite results
+                    overall_suite_results.append(page_cycle_results)
+
+                # so now this is the format that we have (repeated for each pagecycle)
+                # i.e. this is ONE pagecycle:
 
                 # {u'name': u'raptor-speedometer', u'type': u'benchmark', u'measurements':
                 # {u'speedometer': [[{u'AngularJS-TodoMVC/DeletingAllItems': [147.3000000000011,
                 # 149.95999999999913, 143.29999999999927, 150.34000000000378, 257.6999999999971],
                 # u'Inferno-TodoMVC/CompletingAllItems/Sync': [88.03999999999996,#
                 # 85.60000000000036, 94.18000000000029, 95.19999999999709, 86.47999999999593],
                 # u'AngularJS-TodoMVC': [518.2400000000016, 525.8199999999997, 610.5199999999968,
                 # 532.8200000000215, 640.1800000000003], ...(repeated for each index/subtest)}]]},
                 # u'browser': u'Firefox 62.0a1 20180528123052', u'lower_is_better': False, u'page':
                 # u'http://localhost:55019/Speedometer/index.html?raptor', u'unit': u'score',
                 # u'alert_threshold': 2}
 
-                for page_cycle in test.measurements['speedometer']:
-                    page_cycle_results = page_cycle[0]
+                # each benchmark 'index' becomes a subtest; each pagecycle / iteration
+                # of the test has multiple values per index/subtest
 
+                for page_cycle_results in overall_suite_results:
                     for sub, replicates in page_cycle_results.iteritems():
                         # for each pagecycle, replicates are appended to each subtest
                         # so if it doesn't exist the first time create the subtest entry
                         existing = False
                         for existing_sub in subtests:
                             if existing_sub['name'] == sub:
                                 # pagecycle, subtest already there, so append the replicates
                                 existing_sub['replicates'].extend(replicates)
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -2,17 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # speedometer benchmark for firefox and chrome
 
 [DEFAULT]
 type =  benchmark
 test_url = http://localhost:<port>/Speedometer/index.html?raptor
-page_cycles = 1
+page_cycles = 5
 page_timeout = 120000
 unit = score
 lower_is_better = false
 alert_threshold = 2.0
 
 [raptor-speedometer-firefox]
 apps = firefox
 
--- a/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
+++ b/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
@@ -81,17 +81,17 @@
                 }
                 fullNames = new Array;
                 for (var fullName in measuredValuesByFullName) {
                     for (var count=0; count < this.iterationCount; count++) {
                         fullNames.push(fullName);
                     }
                 }
                 if (location.search == '?raptor') {
-                    _data = ['raptor-benchmark', 'speedometer', measuredValuesByFullName];
+                    _data = ['raptor-benchmark', 'speedometer', values, fullNames];
                     window.postMessage(_data, '*');
                 } else {
                     tpRecordTime(values.join(','), 0, fullNames.join(','));
                 }
             } else {
                 for (var i = 0; i < fullNames.length; i++) {
                     var values = measuredValuesByFullName[fullNames[i]];
                     PerfTestRunner.reportValues(createTest(fullNames[i], values.aggregator, i + 1 == fullNames.length), values);