Bug 1373739 - Set up automated test running for WPT in headless mode. r?jgraham
Currently these are enabled for Linux and Windows, using the corresponding WPT
symbols with an "H" suffix to match the Mn->MnH convention followed by the
Marionette tests.
For Linux, the tests are set to run with a simulated 1600x1200 screen
resolution, which matches the Docker configuration. The default 1366x768 for
headless mode results in some unexpected failures due to windows not being
able to reach the full sizes the web platform tests expect.
For Windows, the simulated screen resolution is set to 1024x768, which is what
I've determined to be the resolution that Windows VMs run under within our
testing infrastructure. Setting a different screen resolution causes some web
platform tests to unexpectedly pass (in particular, several in
open-features-non-integer-height.html).
A "headless" variable is added to the environment that the WPT metadata
(.ini) files can access, indicating whether or not the test is running in
headless mode. This is used to mark the reftest bdi-neutral-wrapped.html as an
expected pass on Linux under headless mode: it normally fails (spuriously) due
to differences in scrollbar sizing, but scrollbars aren't rendered in headless
mode.
MozReview-Commit-ID: 3usazOoxx3q
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -40,16 +40,21 @@ common-tests:
- telemetry-tests-client
- xpcshell
web-platform-tests:
- web-platform-tests
- web-platform-tests-reftests
- web-platform-tests-wdspec
+web-platform-tests-headless:
+ - web-platform-tests-headless
+ - web-platform-tests-reftests-headless
+ - web-platform-tests-wdspec-headless
+
opt-only-tests:
- mochitest-valgrind
talos:
- talos-chrome
- talos-dromaeojs
- talos-g1
- talos-g2
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -1764,16 +1764,57 @@ web-platform-tests:
macosx.*:
- web_platform_tests/prod_config.py
default:
- web_platform_tests/prod_config.py
- remove_executables.py
extra-options:
- --test-type=testharness
+web-platform-tests-headless:
+ description: "Web platform test headless run"
+ suite: web-platform-tests
+ treeherder-symbol: tc-W(wptH)
+ chunks:
+ by-test-platform:
+ macosx64/opt: 5
+ macosx64/debug: 10
+ default: 12
+ e10s:
+ by-test-platform:
+ macosx.*: true
+ default: both
+ max-run-time: 7200
+ instance-size: xlarge
+ docker-image: {"in-tree": "desktop1604-test"}
+ run-on-projects: [] # disabled pending releng approval
+ mozharness:
+ script: web_platform_tests.py
+ no-read-buildbot-config: true
+ chunked: true
+ config:
+ by-test-platform:
+ windows.*:
+ - web_platform_tests/prod_config_windows_taskcluster.py
+ macosx.*:
+ - web_platform_tests/prod_config.py
+ default:
+ - web_platform_tests/prod_config.py
+ - remove_executables.py
+ extra-options:
+ by-test-platform:
+ windows.*:
+ - --test-type=testharness
+ - --headless
+ - --headless-width=1024
+ - --headless-height=768
+ default:
+ - --test-type=testharness
+ - --headless
+
web-platform-tests-reftests:
description: "Web platform reftest run"
suite: web-platform-tests-reftests
treeherder-symbol: tc-W(Wr)
max-run-time: 5400
instance-size: xlarge
chunks:
by-test-platform:
@@ -1795,16 +1836,55 @@ web-platform-tests-reftests:
macosx.*:
- web_platform_tests/prod_config.py
default:
- web_platform_tests/prod_config.py
- remove_executables.py
extra-options:
- --test-type=reftest
+web-platform-tests-reftests-headless:
+ description: "Web platform reftest headless run"
+ suite: web-platform-tests-reftests
+ treeherder-symbol: tc-W(WrH)
+ max-run-time: 5400
+ instance-size: xlarge
+ chunks:
+ by-test-platform:
+ macosx.*: 1
+ windows.*: 1
+ default: 6
+ e10s:
+ by-test-platform:
+ macosx.*: true
+ default: both
+ docker-image: {"in-tree": "desktop1604-test"}
+ run-on-projects: [] # disabled pending releng approval
+ mozharness:
+ script: web_platform_tests.py
+ no-read-buildbot-config: true
+ config:
+ by-test-platform:
+ windows.*:
+ - web_platform_tests/prod_config_windows_taskcluster.py
+ macosx.*:
+ - web_platform_tests/prod_config.py
+ default:
+ - web_platform_tests/prod_config.py
+ - remove_executables.py
+ extra-options:
+ by-test-platform:
+ windows.*:
+ - --test-type=reftest
+ - --headless
+ - --headless-width=1024
+ - --headless-height=768
+ default:
+ - --test-type=reftest
+
web-platform-tests-wdspec:
description: "Web platform webdriver-spec run"
suite: web-platform-tests-wdspec
treeherder-symbol: tc-W(Wd)
max-run-time: 5400
instance-size: xlarge
docker-image: {"in-tree": "desktop1604-test"}
mozharness:
@@ -1817,16 +1897,47 @@ web-platform-tests-wdspec:
macosx.*:
- web_platform_tests/prod_config.py
default:
- web_platform_tests/prod_config.py
- remove_executables.py
extra-options:
- --test-type=wdspec
+web-platform-tests-wdspec-headless:
+ description: "Web platform webdriver-spec headless run"
+ suite: web-platform-tests-wdspec
+ treeherder-symbol: tc-W(WdH)
+ max-run-time: 5400
+ instance-size: xlarge
+ docker-image: {"in-tree": "desktop1604-test"}
+ run-on-projects: [] # disabled pending releng approval
+ mozharness:
+ script: web_platform_tests.py
+ no-read-buildbot-config: true
+ config:
+ by-test-platform:
+ windows.*:
+ - web_platform_tests/prod_config_windows_taskcluster.py
+ macosx.*:
+ - web_platform_tests/prod_config.py
+ default:
+ - web_platform_tests/prod_config.py
+ - remove_executables.py
+ extra-options:
+ by-test-platform:
+ windows.*:
+ - --test-type=wdspec
+ - --headless
+ - --headless-width=1024
+ - --headless-height=768
+ default:
+ - --test-type=wdspec
+ - --headless
+
xpcshell:
description: "xpcshell test run"
suite:
by-test-platform:
linux64-jsdcov/opt: xpcshell-coverage
default: xpcshell
treeherder-symbol: tc-X(X)
run-on-projects:
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -57,16 +57,34 @@ class WebPlatformTest(TestingMixin, Merc
"help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."}
],
[["--enable-webrender"], {
"action": "store_true",
"dest": "enable_webrender",
"default": False,
"help": "Tries to enable the WebRender compositor."}
],
+ [["--headless"], {
+ "action": "store_true",
+ "dest": "headless",
+ "default": False,
+ "help": "Run tests in headless mode."}
+ ],
+ [["--headless-width"], {
+ "action": "store",
+ "dest": "headless_width",
+ "default": "1600",
+ "help": "Specify headless fake screen width (default: 1600)."}
+ ],
+ [["--headless-height"], {
+ "action": "store",
+ "dest": "headless_height",
+ "default": "1200",
+ "help": "Specify headless fake screen height (default: 1200)."}
+ ],
[["--single-stylo-traversal"], {
"action": "store_true",
"dest": "single_stylo_traversal",
"default": False,
"help": "Forcibly enable single thread traversal in Stylo with STYLO_THREADS=1"}
],
[["--enable-stylo"], {
"action": "store_true",
@@ -258,16 +276,20 @@ class WebPlatformTest(TestingMixin, Merc
env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
env['RUST_BACKTRACE'] = '1'
if self.config['allow_software_gl_layers']:
env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
if self.config['enable_webrender']:
env['MOZ_WEBRENDER'] = '1'
+ if self.config['headless']:
+ env['MOZ_HEADLESS'] = '1'
+ env['MOZ_HEADLESS_WIDTH'] = self.config['headless_width']
+ env['MOZ_HEADLESS_HEIGHT'] = self.config['headless_height']
if self.config['single_stylo_traversal']:
env['STYLO_THREADS'] = '1'
else:
env['STYLO_THREADS'] = '4'
if self.config['enable_stylo']:
env['STYLO_FORCE_ENABLED'] = '1'
--- a/testing/web-platform/meta/html/semantics/text-level-semantics/the-bdi-element/bdi-neutral-wrapped.html.ini
+++ b/testing/web-platform/meta/html/semantics/text-level-semantics/the-bdi-element/bdi-neutral-wrapped.html.ini
@@ -1,4 +1,5 @@
[bdi-neutral-wrapped.html]
type: reftest
expected:
+ if headless: PASS
if os == "linux": FAIL
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -117,17 +117,18 @@ def env_options():
return {"host": "127.0.0.1",
"external_host": "web-platform.test",
"bind_hostname": "false",
"certificate_domain": "web-platform.test",
"supports_debugger": True}
def run_info_extras(**kwargs):
- return {"e10s": kwargs["gecko_e10s"]}
+ return {"e10s": kwargs["gecko_e10s"],
+ "headless": "MOZ_HEADLESS" in os.environ}
def update_properties():
return (["debug", "stylo", "e10s", "os", "version", "processor", "bits"],
{"debug", "e10s", "stylo"})
class FirefoxBrowser(Browser):