Bug 1436827 - add new raptor 'gdocs' pageload test suite (for google docs/slides/sheets); r?igoldan
MozReview-Commit-ID: EfhBJPgpyRc
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -20,157 +20,175 @@ job-defaults:
- raptor/windows_config.py
windows10-64-ccov/debug:
- raptor/windows_vm_config.py
linux64-ccov/opt:
- raptor/linux64_config_taskcluster.py
default:
- raptor/linux_config.py
-raptor-firefox-tp6:
- description: "Raptor Firefox tp6"
- try-name: raptor-firefox-tp6
+raptor-tp6-firefox:
+ description: "Raptor tp6 on Firefox"
+ try-name: raptor-tp6-firefox
treeherder-symbol: Rap(tp6)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1200
mozharness:
extra-options:
- --test=raptor-tp6
-raptor-firefox-speedometer:
- description: "Raptor Firefox Speedometer"
- try-name: raptor-firefox-speedometer
+raptor-tp6-chrome:
+ description: "Raptor tp6 on Chrome"
+ try-name: raptor-tp6-chrome
+ treeherder-symbol: Rap-C(tp6)
+ run-on-projects: ['try']
+ max-run-time: 1200
+ mozharness:
+ extra-options:
+ - --test=raptor-tp6
+ - --app=chrome
+
+raptor-speedometer-firefox:
+ description: "Raptor Speedometer on Firefox"
+ try-name: raptor-speedometer-firefox
treeherder-symbol: Rap(sp)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-speedometer
-raptor-firefox-stylebench:
- description: "Raptor Firefox StyleBench"
- try-name: raptor-firefox-stylebench
- treeherder-symbol: Rap(sb)
- run-on-projects: ['try', 'mozilla-central']
- max-run-time: 1800
- mozharness:
- extra-options:
- - --test=raptor-stylebench
-
-raptor-firefox-sunspider:
- description: "Raptor Firefox SunSpider"
- try-name: raptor-firefox-sunspider
- treeherder-symbol: Rap(ss)
- run-on-projects: ['try', 'mozilla-central']
- max-run-time: 1800
- mozharness:
- extra-options:
- - --test=raptor-sunspider
-
-raptor-firefox-motionmark-htmlsuite:
- description: "Raptor Firefox MotionMark HtmlSuite"
- try-name: raptor-firefox-motionmark-htmlsuite
- treeherder-symbol: Rap(mm-h)
- run-on-projects: ['try', 'mozilla-central']
- max-run-time: 1800
- mozharness:
- extra-options:
- - --test=raptor-motionmark-htmlsuite
-
-raptor-firefox-motionmark-animometer:
- description: "Raptor Firefox MotionMark Animometer"
- try-name: raptor-firefox-motionmark-animometer
- treeherder-symbol: Rap(mm-a)
- run-on-projects: ['try', 'mozilla-central']
- max-run-time: 1800
- mozharness:
- extra-options:
- - --test=raptor-motionmark-animometer
-
-raptor-firefox-webaudio:
- description: "Raptor Firefox WebAudio benchmark"
- try-name: raptor-firefox-webaudio
- treeherder-symbol: Rap(wa)
- run-on-projects: ['try', 'mozilla-central']
- max-run-time: 1800
- mozharness:
- extra-options:
- - --test=raptor-webaudio
-
-raptor-chrome-tp6:
- description: "Raptor Chrome tp6"
- try-name: raptor-chrome-tp6
- treeherder-symbol: Rap-C(tp6)
- run-on-projects:
- by-test-platform:
- .*-qr/.*: ['try']
- default: ['try']
- max-run-time: 1200
- mozharness:
- extra-options:
- - --test=raptor-tp6
- - --app=chrome
-
-raptor-chrome-speedometer:
- description: "Raptor Chrome Speedometer"
- try-name: raptor-chrome-speedometer
+raptor-speedometer-chrome:
+ description: "Raptor Speedometer on Chrome"
+ try-name: raptor-speedometer-chrome
treeherder-symbol: Rap-C(sp)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1500
mozharness:
extra-options:
- --test=raptor-speedometer
- --app=chrome
-raptor-chrome-stylebench:
- description: "Raptor Chrome StyleBench"
- try-name: raptor-chrome-stylebench
+raptor-stylebench-firefox:
+ description: "Raptor StyleBench on Firefox"
+ try-name: raptor-stylebench-firefox
+ treeherder-symbol: Rap(sb)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-stylebench
+
+raptor-stylebench-chrome:
+ description: "Raptor StyleBench on Chrome"
+ try-name: raptor-stylebench-chrome
treeherder-symbol: Rap-C(sb)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-stylebench
- --app=chrome
-raptor-chrome-sunspider:
- description: "Raptor Chrome SunSpider"
- try-name: raptor-chrome-sunspider
- treeherder-symbol: Rap-C(ss)
+raptor-motionmark-htmlsuite-firefox:
+ description: "Raptor MotionMark HtmlSuite on Firefox"
+ try-name: raptor-motionmark-htmlsuite-firefox
+ treeherder-symbol: Rap(mm-h)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- - --test=raptor-sunspider
- - --app=chrome
+ - --test=raptor-motionmark-htmlsuite
-raptor-chrome-motionmark-htmlsuite:
- description: "Raptor Chrome MotionMark HtmlSuite"
- try-name: raptor-chrome-motionmark-htmlsuite
+raptor-motionmark-htmlsuite-chrome:
+ description: "Raptor MotionMark HtmlSuite on Chrome"
+ try-name: raptor-motionmark-htmlsuite-chrome
treeherder-symbol: Rap-C(mm-h)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-htmlsuite
- --app=chrome
-raptor-chrome-motionmark-animometer:
- description: "Raptor Chrome MotionMark Animometer"
- try-name: raptor-chrome-motionmark-animometer
+raptor-motionmark-animometer-firefox:
+ description: "Raptor MotionMark Animometer on Firefox"
+ try-name: raptor-motionmark-animometer-firefox
+ treeherder-symbol: Rap(mm-a)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-motionmark-animometer
+
+raptor-motionmark-animometer-chrome:
+ description: "Raptor MotionMark Animometer on Chrome"
+ try-name: raptor-motionmark-animometer-chrome
treeherder-symbol: Rap-C(mm-a)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-motionmark-animometer
- --app=chrome
-raptor-chrome-webaudio:
- description: "Raptor Chrome WebAudio benchmark"
- try-name: raptor-chrome-webaudio
+raptor-webaudio-firefox:
+ description: "Raptor WebAudio on Firefox"
+ try-name: raptor-webaudio-firefox
+ treeherder-symbol: Rap(wa)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-webaudio
+
+raptor-webaudio-chrome:
+ description: "Raptor WebAudio on Chrome"
+ try-name: raptor-webaudio-chrome
treeherder-symbol: Rap-C(wa)
run-on-projects: ['try', 'mozilla-central']
max-run-time: 1800
mozharness:
extra-options:
- --test=raptor-webaudio
- --app=chrome
+
+raptor-gdocs-firefox:
+ description: "Raptor gdocs on Firefox"
+ try-name: raptor-gdocs-firefox
+ treeherder-symbol: Rap(gdocs)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-gdocs
+
+raptor-gdocs-chrome:
+ description: "Raptor gdocs on Chrome"
+ try-name: raptor-gdocs-chrome
+ treeherder-symbol: Rap-C(gdocs)
+ run-on-projects: ['try']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-gdocs
+ - --app=chrome
+
+raptor-sunspider-firefox:
+ description: "Raptor SunSpider on Firefox"
+ try-name: raptor-sunspider-firefox
+ treeherder-symbol: Rap(ss)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-sunspider
+
+raptor-sunspider-chrome:
+ description: "Raptor SunSpider on Chrome"
+ try-name: raptor-sunspider-chrome
+ treeherder-symbol: Rap-C(ss)
+ run-on-projects: ['try', 'mozilla-central']
+ max-run-time: 1800
+ mozharness:
+ extra-options:
+ - --test=raptor-sunspider
+ - --app=chrome
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -76,28 +76,32 @@ talos:
- talos-tp6-stylo-threads
- talos-tps
- talos-speedometer
- talos-motionmark
- talos-h1
- talos-h2
raptor:
- - raptor-firefox-tp6
- - raptor-firefox-speedometer
- - raptor-firefox-stylebench
- - raptor-firefox-sunspider
- - raptor-firefox-motionmark-htmlsuite
- - raptor-firefox-motionmark-animometer
- - raptor-firefox-webaudio
- - raptor-chrome-tp6
- - raptor-chrome-speedometer
- - raptor-chrome-stylebench
- - raptor-chrome-sunspider
- - raptor-chrome-webaudio
+ - raptor-tp6-firefox
+ - raptor-tp6-chrome
+ - raptor-speedometer-firefox
+ - raptor-speedometer-chrome
+ - raptor-stylebench-firefox
+ - raptor-stylebench-chrome
+ - raptor-motionmark-htmlsuite-firefox
+ - raptor-motionmark-htmlsuite-chrome
+ - raptor-motionmark-animometer-firefox
+ - raptor-motionmark-animometer-chrome
+ - raptor-webaudio-firefox
+ - raptor-webaudio-chrome
+ - raptor-gdocs-firefox
+ - raptor-gdocs-chrome
+ - raptor-sunspider-firefox
+ - raptor-sunspider-chrome
awsy:
- awsy
- awsy-base
awsy-stylo-sequential:
- awsy-stylo-sequential
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-gdocs.manifest
@@ -0,0 +1,9 @@
+[
+ {
+ "filename": "mitmproxy-recordings-raptor-gdocs.zip",
+ "size": 26179496,
+ "digest": "eb32218815d3c2187807f3a18d6df1657e01a215820deaaa7a035f06d180780ac9652479ac7db1d8cde3728a890521aff2e0a5c4738304966b2a93ce1b7f33bb",
+ "algorithm": "sha512",
+ "unpack": true
+ }
+]
\ No newline at end of file
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,8 +1,9 @@
# raptor tests
[include:tests/raptor-tp6.ini]
[include:tests/raptor-speedometer.ini]
[include:tests/raptor-stylebench.ini]
[include:tests/raptor-sunspider.ini]
[include:tests/raptor-motionmark-htmlsuite.ini]
[include:tests/raptor-motionmark-animometer.ini]
[include:tests/raptor-webaudio.ini]
+[include:tests/raptor-gdocs.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-gdocs.ini
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor gdocs
+
+[DEFAULT]
+type = pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-gdocs.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 30000
+
+[raptor-google-docs-firefox]
+apps = firefox
+test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
+playback_recordings = google-docs.mp
+measure = fnbpaint
+
+[raptor-google-sheets-firefox]
+apps = firefox
+test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing
+playback_recordings = google-sheets.mp
+measure = fnbpaint
+
+[raptor-google-slides-firefox]
+apps = firefox
+test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+playback_recordings = google-slides.mp
+measure = fnbpaint
+
+[raptor-google-docs-chrome]
+apps = chrome
+test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
+playback_recordings = google-docs.mp
+measure = fcp
+
+[raptor-google-sheets-chrome]
+apps = chrome
+test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing
+playback_recordings = google-sheets.mp
+measure = fcp
+
+[raptor-google-slides-chrome]
+apps = chrome
+test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+playback_recordings = google-slides.mp
+measure = fcp
--- a/testing/raptor/raptor/tests/raptor-tp6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6.ini
@@ -10,58 +10,58 @@ playback = mitmproxy
playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
python3_win_manifest = python3{x64}.manifest
playback_pageset_manifest = mitmproxy-recordings-raptor-tp6.manifest
page_cycles = 25
unit = ms
lower_is_better = true
alert_threshold = 2.0
-[raptor-firefox-tp6-amazon]
+[raptor-tp6-amazon-firefox]
apps = firefox
test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
playback_recordings = amazon.mp
measure = fnbpaint
-[raptor-firefox-tp6-facebook]
+[raptor-tp6-facebook-firefox]
apps = firefox
test_url = https://www.facebook.com
playback_recordings = facebook.mp
measure = fnbpaint
-[raptor-firefox-tp6-google]
+[raptor-tp6-google-firefox]
apps = firefox
# note: use the full url as the first part (without '&cad=h') redirects
# to the url + '&cad=h'; that redirection causes measure.js content
# to be loaded into that page also; resulting in 2 fnbpaint values etc.
test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
playback_recordings = google-search.mp
measure = fnbpaint
-[raptor-firefox-tp6-youtube]
+[raptor-tp6-youtube-firefox]
apps = firefox
test_url = https://www.youtube.com
playback_recordings = youtube.mp
measure = fnbpaint
-[raptor-chrome-tp6-amazon]
+[raptor-tp6-amazon-chrome]
apps = chrome
test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
playback_recordings = amazon.mp
measure = fcp
-[raptor-chrome-tp6-facebook]
+[raptor-tp6-facebook-chrome]
apps = chrome
test_url = https://www.facebook.com
playback_recordings = facebook.mp
measure = fcp
-[raptor-chrome-tp6-google]
+[raptor-tp6-google-chrome]
apps = chrome
test_url = https://www.google.com/#hl=en&q=barack+obama
playback_recordings = google-search.mp
measure = fcp
-[raptor-chrome-tp6-youtube]
+[raptor-tp6-youtube-chrome]
apps = chrome
test_url = https://www.youtube.com
playback_recordings = youtube.mp
measure = fcp