Bug 1367041 - Add |mach wpt| support for Chrome, Edge and Servo, r=AutomatedTester
Using the wptrun infrastructure from upstream, it is now posible to
make it easy to run web-platform-tests in other browsers. The syntax
used is
mach wpt --product [chrome|servo|edge] [tests]
This will try to use the selected product; possibly prompting to
install dependencies like the WebDriver implementation. For servo if
the install isn't on the PATH then --binary can be used to point to
the actual location.
Because manifest metadata is kept in the same directory as expectation
data and we don't want to reuse Firefox expectation data for other
browsers, a new products subdirectory is introduced and added to the
ignore files. This will contain a subdirectory for each product into
which a copy of the test manifest is placed. It may also be used to
store any expectation data for the other products, in the same way as
testing/web-platform/meta.
MozReview-Commit-ID: 8fdCnha5t2F
--- a/.gitignore
+++ b/.gitignore
@@ -85,16 +85,19 @@ devtools/**/node_modules
GTAGS
GRTAGS
GSYMS
GPATH
# Git clone directory for updating web-platform-tests
testing/web-platform/sync/
+# Third party metadata for web-platform-tests
+testing/web-platform/products/
+
# Android Gradle artifacts.
mobile/android/gradle/.gradle
# XCode project cruft
embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/project.xcworkspace/xcuserdata
embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/xcuserdata
# Ignore mozharness execution files
--- a/.hgignore
+++ b/.hgignore
@@ -93,16 +93,19 @@
GTAGS
GRTAGS
GSYMS
GPATH
# Git clone directory for updating web-platform-tests
^testing/web-platform/sync/
+# Third party metadata for web-platform-tests
+^testing/web-platform/products/
+
# Android Gradle artifacts.
^mobile/android/gradle/.gradle
# XCode project cruft
^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/project.xcworkspace/xcuserdata
^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/xcuserdata
# Ignore mozharness execution files
--- a/testing/web-platform/README.md
+++ b/testing/web-platform/README.md
@@ -230,50 +230,19 @@ In order to update the manifest it is re
web-platform-tests --manifest-update`. This rescans the test directory
looking for new, removed, or altered tests.
Running Tests In Other Browsers
-------------------------------
web-platform-tests is cross browser, and the runner is compatible with
multiple browsers. Therefore it's possible to check the behaviour of
-tests in other browsers. This is somewhat more involved than running
-them in Firefox since extra dependencies may be required. For example
-to test in Chrome:
-
-1. Download the chromedriver binary and place it somewhere sensible
- e.g. `~/bin`
+tests in other browsers. By default Chrome, Edge and Servo are
+supported. In order to run the tests in these browsers use the
+`--product` argument to wptrunner:
-2. In your gecko source tree activate the virtualenv created by mach,
- since this has most dependencies already installed. This is typically
- in objdir/_virtualenv and is activated via e.g.
-
- source objdir/_virtualenv/bin/activate
-
-3. Install the extra requirements:
-
- cd testing/web-platform/harness
- pip install -r requirements_chrome.txt
-
-4. Edit the config file `testing/web-platform/wptrunner.ini` so that
- Chrome support is enabled by changing the section that reads:
+ mach wpt --product chrome dom/historical.html
- [products]
- firefox =
-
- to read
-
- [products]
- firefox =
- chrome =
-
- (alternatively create a new config file elsewhere and use the
- `--config` option to `runtests.py` to point wptrunner at this config
- file).
-
-5. Run `runtests.py` using the location of chromedriver as
- the binary:
-
- cd testing/web-platform
- python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=-
-
-By default this will use the same test checkout and metadata as are in
-the Gecko tree, so it's easy to compare behaviour relative to Firefox.
+By default these browsers run without expectation metadata, but it can
+be added in the `testing/web-platform/products/<product>`
+directory. To run with the same metadata as for Firefox (so that
+differences are reported as unexpected results), pass `--meta
+testing/web-platform/meta` to the mach command.
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -23,17 +23,17 @@ from mach.decorators import (
# This should probably be consolidated with similar classes in other test
# runners.
class InvalidTestPathError(Exception):
"""Exception raised when the test path is not valid."""
class WebPlatformTestsRunner(MozbuildObject):
"""Run web platform tests."""
- def setup_kwargs(self, kwargs):
+ def setup_kwargs_firefox(self, kwargs):
from wptrunner import wptcommandline
build_path = os.path.join(self.topobjdir, 'build')
if build_path not in sys.path:
sys.path.append(build_path)
if kwargs["config"] is None:
kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
@@ -62,20 +62,60 @@ class WebPlatformTestsRunner(MozbuildObj
if kwargs["host_cert_path"] is None:
kwargs["host_cert_path"] = os.path.join(here, "certs", "web-platform.test.pem")
kwargs["capture_stdio"] = True
kwargs = wptcommandline.check_args(kwargs)
+ def setup_kwargs_wptrun(self, kwargs):
+ from wptrunner import wptcommandline
+ here = os.path.join(self.topsrcdir, 'testing', 'web-platform')
+
+ sys.path.insert(0, os.path.join(here, "tests", "tools"))
+
+ import wptrun
+
+ setup_func = {
+ "chrome": wptrun.setup_chrome,
+ "edge": wptrun.setup_edge,
+ "servo": wptrun.setup_servo,
+ }[kwargs["product"]]
+
+ setup_func(wptrun.virtualenv.Virtualenv(self.virtualenv_manager.virtualenv_root),
+ kwargs,
+ True)
+
+ kwargs["tests_root"] = os.path.join(here, "tests")
+
+ if kwargs["metadata_root"] is None:
+ metadir = os.path.join(here, "products", kwargs["product"])
+ if not os.path.exists(metadir):
+ os.makedirs(metadir)
+ kwargs["metadata_root"] = metadir
+
+ src_manifest = os.path.join(here, "meta", "MANIFEST.json")
+ dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
+
+ if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
+ with open(src_manifest) as src, open(dest_manifest, "w") as dest:
+ dest.write(src.read())
+
+ kwargs = wptcommandline.check_args(kwargs)
+
def run_tests(self, **kwargs):
from wptrunner import wptrunner
- self.setup_kwargs(kwargs)
+ if kwargs["product"] in ["firefox", None]:
+ self.setup_kwargs_firefox(kwargs)
+ elif kwargs["product"] in ("chrome", "edge", "servo"):
+ self.setup_kwargs_wptrun(kwargs)
+ else:
+ raise ValueError("Unknown product %s" % kwargs["product"])
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
result = wptrunner.run_tests(**kwargs)
return int(not result)
def list_test_groups(self, **kwargs):
from wptrunner import wptrunner
@@ -253,17 +293,17 @@ class WPTManifestUpdater(MozbuildObject)
from wptrunner import wptlogging
logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
manifestupdate.update(logger, wpt_dir, check_clean)
def create_parser_wpt():
from wptrunner import wptcommandline
- return wptcommandline.create_parser(["firefox"])
+ return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
def create_parser_update():
from update import updatecommandline
return updatecommandline.create_parser()
def create_parser_reduce():
from wptrunner import wptcommandline
return wptcommandline.create_parser_reduce()
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/products/README
@@ -0,0 +1,3 @@
+This directory is for storing metadata files when running
+web-platform-tests with non-firefox browsers. By default all
+subdirectories here are ignored in VCS.
\ No newline at end of file
--- a/testing/web-platform/tests/tools/wptrunner/requirements_firefox.txt
+++ b/testing/web-platform/tests/tools/wptrunner/requirements_firefox.txt
@@ -1,5 +1,6 @@
marionette_driver >= 0.4
mozprofile >= 0.21
mozprocess >= 0.19
mozcrash >= 0.13
mozrunner >= 6.7
+mozleak >= 0.1
--- a/testing/web-platform/wptrunner.ini
+++ b/testing/web-platform/wptrunner.ini
@@ -1,10 +1,13 @@
[products]
firefox =
+chrome =
+edge =
+servo =
[web-platform-tests]
remote_url = https://github.com/w3c/web-platform-tests.git
branch = master
sync_path = sync
[paths]
prefs = ../profiles