Bug 1363428 - Support Marionette reftest implementation in wptrunner, r=maja_zf
Add an InternalReftestImplmentation that runs reftests using the
built-in Marionette reftest primitives rather than screenshots. This
is actiivated using the --internal-reftest switch, although that may
become the default in the future.
MozReview-Commit-ID: 6HxGuBsTITf
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -60,16 +60,17 @@ def check_args(**kwargs):
if kwargs["ssl_type"] != "none":
require_arg(kwargs, "certutil_binary")
def browser_kwargs(test_type, run_info_data, **kwargs):
return {"binary": kwargs["binary"],
"prefs_root": kwargs["prefs_root"],
"extra_prefs": kwargs["extra_prefs"],
+ "test_type": test_type,
"debug_info": kwargs["debug_info"],
"symbols_path": kwargs["symbols_path"],
"stackwalk_binary": kwargs["stackwalk_binary"],
"certutil_binary": kwargs["certutil_binary"],
"ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
"e10s": kwargs["gecko_e10s"],
"stackfix_dir": kwargs["stackfix_dir"],
"binary_args": kwargs["binary_args"],
@@ -83,16 +84,19 @@ def browser_kwargs(test_type, run_info_d
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
executor_kwargs = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
executor_kwargs["close_after_done"] = test_type != "reftest"
executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type,
run_info_data,
**kwargs)
+ if test_type == "reftest":
+ executor_kwargs["reftest_internal"] = kwargs["reftest_internal"]
+ executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
if test_type == "wdspec":
executor_kwargs["binary"] = kwargs["binary"]
executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
executor_kwargs["webdriver_args"] = kwargs.get("webdriver_args")
fxOptions = {}
if kwargs["binary"]:
fxOptions["binary"] = kwargs["binary"]
if kwargs["binary_args"]:
@@ -126,23 +130,24 @@ def update_properties():
{"debug", "e10s", "stylo"})
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
shutdown_timeout = 60
- def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None,
+ def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, stackfix_dir=None,
binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
+ self.test_type = test_type
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
@@ -180,21 +185,23 @@ class FirefoxBrowser(Browser):
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False,
- "dom.send_after_paint_to_content": True,
- "layout.interruptible-reflow.enabled": False})
+ "dom.send_after_paint_to_content": True})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
+ if self.test_type == "reftest":
+ self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
+
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
@@ -210,16 +210,22 @@ class RefTestImplementation(object):
self.executor = executor
# Cache of url:(screenshot hash, screenshot). Typically the
# screenshot is None, but we set this value if a test fails
# and the screenshot was taken from the cache so that we may
# retrieve the screenshot from the cache directly in the future
self.screenshot_cache = self.executor.screenshot_cache
self.message = None
+ def setup(self):
+ pass
+
+ def teardown(self):
+ pass
+
@property
def logger(self):
return self.executor.logger
def get_hash(self, test, viewport_size, dpi):
timeout = test.timeout * self.timeout_multiplier
key = (test.url, viewport_size, dpi)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -462,53 +462,70 @@ class MarionetteTestharnessExecutor(Test
rv = marionette.execute_async_script(script, new_sandbox=False)
return rv
class MarionetteRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, close_after_done=True,
- debug_info=None, **kwargs):
-
+ debug_info=None, reftest_internal=False,
+ reftest_screenshot="unexpected",
+ **kwargs):
"""Marionette-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
server_config,
screenshot_cache=screenshot_cache,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = MarionetteProtocol(self, browser)
- self.implementation = RefTestImplementation(self)
+ self.implementation = (InternalRefTestImplementation
+ if reftest_internal
+ else RefTestImplementation)(self)
+ self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
+ reftest_internal else {})
+
self.close_after_done = close_after_done
self.has_window = False
self.original_pref_values = {}
with open(os.path.join(here, "reftest.js")) as f:
self.script = f.read()
with open(os.path.join(here, "reftest-wait.js")) as f:
self.wait_script = f.read()
+ def setup(self, runner):
+ super(self.__class__, self).setup(runner)
+ self.implementation.setup(**self.implementation_kwargs)
+
+ def teardown(self):
+ self.implementation.teardown()
+ handle = self.protocol.marionette.window_handles[0]
+ self.protocol.marionette.switch_to_window(handle)
+ super(self.__class__, self).teardown()
+
def is_alive(self):
return self.protocol.is_alive
def on_environment_change(self, new_environment):
self.protocol.on_environment_change(self.last_environment, new_environment)
def do_test(self, test):
- if self.close_after_done and self.has_window:
- self.protocol.marionette.close()
- self.protocol.marionette.switch_to_window(
- self.protocol.marionette.window_handles[-1])
- self.has_window = False
+ if not isinstance(self.implementation, InternalRefTestImplementation):
+ if self.close_after_done and self.has_window:
+ self.protocol.marionette.close()
+ self.protocol.marionette.switch_to_window(
+ self.protocol.marionette.window_handles[-1])
+ self.has_window = False
- if not self.has_window:
- self.protocol.marionette.execute_script(self.script)
- self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
- self.has_window = True
+ if not self.has_window:
+ self.protocol.marionette.execute_script(self.script)
+ self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
+ self.has_window = True
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def screenshot(self, test, viewport_size, dpi):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
assert dpi is None
@@ -531,16 +548,59 @@ class MarionetteRefTestExecutor(RefTestE
screenshot = marionette.screenshot(full=False)
# strip off the data:img/png, part of the url
if screenshot.startswith("data:image/png;base64,"):
screenshot = screenshot.split(",", 1)[1]
return screenshot
+class InternalRefTestImplementation(object):
+ def __init__(self, executor):
+ self.timeout_multiplier = executor.timeout_multiplier
+ self.executor = executor
+
+ @property
+ def logger(self):
+ return self.executor.logger
+
+ def setup(self, screenshot="unexpected"):
+ data = {"screenshot": screenshot}
+ if self.executor.queue_metadata is not None:
+ data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
+ for key, value in self.executor.queue_metadata.get("url_count", {}).iteritems()
+ if value > 1}
+ self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
+ self.executor.protocol.marionette._send_message("reftest:setup", data)
+
+ def run_test(self, test):
+ viewport_size = test.viewport_size
+ dpi = test.dpi
+
+ references = self.get_references(test)
+ rv = self.executor.protocol.marionette._send_message("reftest:run",
+ {"test": self.executor.test_url(test),
+ "references": references,
+ "expected": test.expected(),
+ "timeout": test.timeout * 1000})["value"]
+ return rv
+
+ def get_references(self, node):
+ rv = []
+ for item, relation in node.references:
+ rv.append([self.executor.test_url(item), self.get_references(item), relation])
+ return rv
+
+ def teardown(self):
+ try:
+ self.executor.protocol.marionette._send_message("reftest:teardown", {})
+ self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
+ except socket.error:
+ pass
+
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
self.result = (None, None)
self.session = session
self.path = path
self.timeout = timeout
self.result_flag = threading.Event()
@@ -574,17 +634,17 @@ class WdspecRun(object):
self.result = False, ("ERROR", message)
finally:
self.result_flag.set()
class MarionetteWdspecExecutor(WdspecExecutor):
def __init__(self, browser, server_config, webdriver_binary,
timeout_multiplier=1, close_after_done=True, debug_info=None,
- capabilities=None, webdriver_args=None, binary=None):
+ capabilities=None, webdriver_args=None, binary=None, **kwargs):
self.do_delayed_imports()
WdspecExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.webdriver_binary = webdriver_binary
self.webdriver_args = webdriver_args + ["--binary", binary]
self.capabilities = capabilities
self.protocol = RemoteMarionetteProtocol(self, browser)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -179,16 +179,23 @@ scheme host and port.""")
help="Path to directory containing assertion stack fixing scripts")
gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
default=[], metavar="PREF=VALUE",
help="Defines an extra user preference (overrides those in prefs_root)")
gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
help="Enable leak checking")
gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
help="Number of parallel threads to use for stylo")
+ gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
+ default=None, help="Enable reftest runner implemented inside Marionette")
+ gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
+ help="Disable reftest runner implemented inside Marionette")
+ gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
+ choices=["always", "fail", "unexpected"], default="unexpected",
+ help="With --reftest-internal, when to take a screenshot")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",
default=[], action="append", dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
sauce_group = parser.add_argument_group("Sauce Labs-specific")
sauce_group.add_argument("--sauce-browser", dest="sauce_browser",