Bug 1469280 - Fix raptor speedometer score calculation; r?jmaher
MozReview-Commit-ID: FxyanRA2LLk
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -81,37 +81,69 @@ class Output(object):
new_subtest['unit'] = test.unit
filtered_values = filter.ignore_first(new_subtest['replicates'], 1)
new_subtest['value'] = filter.median(filtered_values)
vals.append(new_subtest['value'])
subtests.append(new_subtest)
- elif test.type == "benchmark":
- # each benchmark 'index' becomes a subtest; each pagecycle / iteration
- # of the test has multiple values per index/subtest
+ elif test.type == "benchmark" and 'speedometer' in test.name:
+
+ # results come from benchmark (via control server) formatted as an array,
+ # that contains results from each pagecycle. Each pagecycle is formatted
+ # as: array[0] contains a list of test values, in sets of 5 (speedometer
+ # has 5 values per single run per subsuite). array[1] contains a list of
+ # of corresponding subsuite names (subsuite name duplicated 5x)
+ # so for each single pagecycle, there are 160 sets of 5 vals (800 vals total)
+ # and there are 800 corresponding subsuite names (160 * 5)
+
+ # first lets convert the pagecycle to a list of dicts - where each dict
+ # item will be { 'subsuite_name': [val1, val2, val3, val4, val5] }
+ # so only one dict entry for each of the 160 unique subsuite names
+
+ overall_suite_results = []
- # this is the format we receive the results in from the benchmark
- # i.e. this is ONE pagecycle of speedometer:
+ for page_cycle in test.measurements['speedometer']:
+ # use an OrderedDict so the order entries are added is persisted
+ # we need the speedometer subsuites to be in the order they're reported
+ from collections import OrderedDict
+ page_cycle_results = OrderedDict()
+
+ page_cycle_vals = page_cycle[0]
+ page_cycle_names = page_cycle[1]
+
+ for count in range(len(page_cycle_vals)):
+ # page_cycle_vals[count] = [val1, val2, val3, val4, val5]
+ # this covers/is for the corresponding 5 subsuite names
+ new_entry = {}
+ new_entry[page_cycle_names[count * 5]] = page_cycle_vals[count]
+ page_cycle_results.update(new_entry)
+
+ # now we have that pagecycle results; add to overall suite results
+ overall_suite_results.append(page_cycle_results)
+
+ # so now this is the format that we have (repeated for each pagecycle)
+ # i.e. this is ONE pagecycle:
# {u'name': u'raptor-speedometer', u'type': u'benchmark', u'measurements':
# {u'speedometer': [[{u'AngularJS-TodoMVC/DeletingAllItems': [147.3000000000011,
# 149.95999999999913, 143.29999999999927, 150.34000000000378, 257.6999999999971],
# u'Inferno-TodoMVC/CompletingAllItems/Sync': [88.03999999999996,#
# 85.60000000000036, 94.18000000000029, 95.19999999999709, 86.47999999999593],
# u'AngularJS-TodoMVC': [518.2400000000016, 525.8199999999997, 610.5199999999968,
# 532.8200000000215, 640.1800000000003], ...(repeated for each index/subtest)}]]},
# u'browser': u'Firefox 62.0a1 20180528123052', u'lower_is_better': False, u'page':
# u'http://localhost:55019/Speedometer/index.html?raptor', u'unit': u'score',
# u'alert_threshold': 2}
- for page_cycle in test.measurements['speedometer']:
- page_cycle_results = page_cycle[0]
+ # each benchmark 'index' becomes a subtest; each pagecycle / iteration
+ # of the test has multiple values per index/subtest
+ for page_cycle_results in overall_suite_results:
for sub, replicates in page_cycle_results.iteritems():
# for each pagecycle, replicates are appended to each subtest
# so if it doesn't exist the first time create the subtest entry
existing = False
for existing_sub in subtests:
if existing_sub['name'] == sub:
# pagecycle, subtest already there, so append the replicates
existing_sub['replicates'].extend(replicates)
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -2,17 +2,17 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# speedometer benchmark for firefox and chrome
[DEFAULT]
type = benchmark
test_url = http://localhost:<port>/Speedometer/index.html?raptor
-page_cycles = 1
+page_cycles = 5
page_timeout = 120000
unit = score
lower_is_better = false
alert_threshold = 2.0
[raptor-speedometer-firefox]
apps = firefox
--- a/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
+++ b/third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js
@@ -81,17 +81,17 @@
}
fullNames = new Array;
for (var fullName in measuredValuesByFullName) {
for (var count=0; count < this.iterationCount; count++) {
fullNames.push(fullName);
}
}
if (location.search == '?raptor') {
- _data = ['raptor-benchmark', 'speedometer', measuredValuesByFullName];
+ _data = ['raptor-benchmark', 'speedometer', values, fullNames];
window.postMessage(_data, '*');
} else {
tpRecordTime(values.join(','), 0, fullNames.join(','));
}
} else {
for (var i = 0; i < fullNames.length; i++) {
var values = measuredValuesByFullName[fullNames[i]];
PerfTestRunner.reportValues(createTest(fullNames[i], values.aggregator, i + 1 == fullNames.length), values);