Bug 1363428 - Support Marionette reftest implementation in wptrunner, r=maja_zf draft
authorJames Graham <james@hoppipolla.co.uk>
Wed, 10 May 2017 11:41:33 +0100
changeset 600213 d7be32e0ac3f4c076605e28a2d13b763b328735c
parent 600212 c96ffa60c0c47fd67926dfc1176e5a42a10f02a5
child 600214 2d525263bb91e466bc49cb57f14e2efe2d7d59d7
push id65688
push userbmo:james@hoppipolla.co.uk
push dateSat, 24 Jun 2017 11:04:46 +0000
reviewersmaja_zf
bugs1363428
milestone56.0a1
Bug 1363428 - Support Marionette reftest implementation in wptrunner, r=maja_zf Add an InternalReftestImplmentation that runs reftests using the built-in Marionette reftest primitives rather than screenshots. This is actiivated using the --internal-reftest switch, although that may become the default in the future. MozReview-Commit-ID: 6HxGuBsTITf
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -60,16 +60,17 @@ def check_args(**kwargs):
     if kwargs["ssl_type"] != "none":
         require_arg(kwargs, "certutil_binary")
 
 
 def browser_kwargs(test_type, run_info_data, **kwargs):
     return {"binary": kwargs["binary"],
             "prefs_root": kwargs["prefs_root"],
             "extra_prefs": kwargs["extra_prefs"],
+            "test_type": test_type,
             "debug_info": kwargs["debug_info"],
             "symbols_path": kwargs["symbols_path"],
             "stackwalk_binary": kwargs["stackwalk_binary"],
             "certutil_binary": kwargs["certutil_binary"],
             "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
             "e10s": kwargs["gecko_e10s"],
             "stackfix_dir": kwargs["stackfix_dir"],
             "binary_args": kwargs["binary_args"],
@@ -83,16 +84,19 @@ def browser_kwargs(test_type, run_info_d
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
                     **kwargs):
     executor_kwargs = base_executor_kwargs(test_type, server_config,
                                            cache_manager, **kwargs)
     executor_kwargs["close_after_done"] = test_type != "reftest"
     executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type,
                                                                    run_info_data,
                                                                    **kwargs)
+    if test_type == "reftest":
+        executor_kwargs["reftest_internal"] = kwargs["reftest_internal"]
+        executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
     if test_type == "wdspec":
         executor_kwargs["binary"] = kwargs["binary"]
         executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
         executor_kwargs["webdriver_args"] = kwargs.get("webdriver_args")
         fxOptions = {}
         if kwargs["binary"]:
             fxOptions["binary"] = kwargs["binary"]
         if kwargs["binary_args"]:
@@ -126,23 +130,24 @@ def update_properties():
             {"debug", "e10s", "stylo"})
 
 
 class FirefoxBrowser(Browser):
     used_ports = set()
     init_timeout = 60
     shutdown_timeout = 60
 
-    def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None,
+    def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                  ca_certificate_path=None, e10s=False, stackfix_dir=None,
                  binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
+        self.test_type = test_type
         self.extra_prefs = extra_prefs
         self.marionette_port = None
         self.runner = None
         self.debug_info = debug_info
         self.profile = None
         self.symbols_path = symbols_path
         self.stackwalk_binary = stackwalk_binary
         self.ca_certificate_path = ca_certificate_path
@@ -180,21 +185,23 @@ class FirefoxBrowser(Browser):
 
         self.profile = FirefoxProfile(locations=locations,
                                       preferences=preferences)
         self.profile.set_preferences({"marionette.port": self.marionette_port,
                                       "dom.disable_open_during_load": False,
                                       "network.dns.localDomains": ",".join(hostnames),
                                       "network.proxy.type": 0,
                                       "places.history.enabled": False,
-                                      "dom.send_after_paint_to_content": True,
-                                      "layout.interruptible-reflow.enabled": False})
+                                      "dom.send_after_paint_to_content": True})
         if self.e10s:
             self.profile.set_preferences({"browser.tabs.remote.autostart": True})
 
+        if self.test_type == "reftest":
+            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
+
         if self.leak_check and kwargs.get("check_leaks", True):
             self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
             if os.path.exists(self.leak_report_file):
                 os.remove(self.leak_report_file)
             env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
         else:
             self.leak_report_file = None
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
@@ -210,16 +210,22 @@ class RefTestImplementation(object):
         self.executor = executor
         # Cache of url:(screenshot hash, screenshot). Typically the
         # screenshot is None, but we set this value if a test fails
         # and the screenshot was taken from the cache so that we may
         # retrieve the screenshot from the cache directly in the future
         self.screenshot_cache = self.executor.screenshot_cache
         self.message = None
 
+    def setup(self):
+        pass
+
+    def teardown(self):
+        pass
+
     @property
     def logger(self):
         return self.executor.logger
 
     def get_hash(self, test, viewport_size, dpi):
         timeout = test.timeout * self.timeout_multiplier
         key = (test.url, viewport_size, dpi)
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -462,53 +462,70 @@ class MarionetteTestharnessExecutor(Test
 
         rv = marionette.execute_async_script(script, new_sandbox=False)
         return rv
 
 
 class MarionetteRefTestExecutor(RefTestExecutor):
     def __init__(self, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
-                 debug_info=None, **kwargs):
-
+                 debug_info=None, reftest_internal=False,
+                 reftest_screenshot="unexpected",
+                 **kwargs):
         """Marionette-based executor for reftests"""
         RefTestExecutor.__init__(self,
                                  browser,
                                  server_config,
                                  screenshot_cache=screenshot_cache,
                                  timeout_multiplier=timeout_multiplier,
                                  debug_info=debug_info)
         self.protocol = MarionetteProtocol(self, browser)
-        self.implementation = RefTestImplementation(self)
+        self.implementation = (InternalRefTestImplementation
+                               if reftest_internal
+                               else RefTestImplementation)(self)
+        self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
+                                      reftest_internal else {})
+
         self.close_after_done = close_after_done
         self.has_window = False
         self.original_pref_values = {}
 
         with open(os.path.join(here, "reftest.js")) as f:
             self.script = f.read()
         with open(os.path.join(here, "reftest-wait.js")) as f:
             self.wait_script = f.read()
 
+    def setup(self, runner):
+        super(self.__class__, self).setup(runner)
+        self.implementation.setup(**self.implementation_kwargs)
+
+    def teardown(self):
+        self.implementation.teardown()
+        handle = self.protocol.marionette.window_handles[0]
+        self.protocol.marionette.switch_to_window(handle)
+        super(self.__class__, self).teardown()
+
     def is_alive(self):
         return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         self.protocol.on_environment_change(self.last_environment, new_environment)
 
     def do_test(self, test):
-        if self.close_after_done and self.has_window:
-            self.protocol.marionette.close()
-            self.protocol.marionette.switch_to_window(
-                self.protocol.marionette.window_handles[-1])
-            self.has_window = False
+        if not isinstance(self.implementation, InternalRefTestImplementation):
+            if self.close_after_done and self.has_window:
+                self.protocol.marionette.close()
+                self.protocol.marionette.switch_to_window(
+                    self.protocol.marionette.window_handles[-1])
+                self.has_window = False
 
-        if not self.has_window:
-            self.protocol.marionette.execute_script(self.script)
-            self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
-            self.has_window = True
+            if not self.has_window:
+                self.protocol.marionette.execute_script(self.script)
+                self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
+                self.has_window = True
 
         result = self.implementation.run_test(test)
         return self.convert_result(test, result)
 
     def screenshot(self, test, viewport_size, dpi):
         # https://github.com/w3c/wptrunner/issues/166
         assert viewport_size is None
         assert dpi is None
@@ -531,16 +548,59 @@ class MarionetteRefTestExecutor(RefTestE
         screenshot = marionette.screenshot(full=False)
         # strip off the data:img/png, part of the url
         if screenshot.startswith("data:image/png;base64,"):
             screenshot = screenshot.split(",", 1)[1]
 
         return screenshot
 
 
+class InternalRefTestImplementation(object):
+    def __init__(self, executor):
+        self.timeout_multiplier = executor.timeout_multiplier
+        self.executor = executor
+
+    @property
+    def logger(self):
+        return self.executor.logger
+
+    def setup(self, screenshot="unexpected"):
+        data = {"screenshot": screenshot}
+        if self.executor.queue_metadata is not None:
+            data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
+                                for key, value in self.executor.queue_metadata.get("url_count", {}).iteritems()
+                                if value > 1}
+        self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
+        self.executor.protocol.marionette._send_message("reftest:setup", data)
+
+    def run_test(self, test):
+        viewport_size = test.viewport_size
+        dpi = test.dpi
+
+        references = self.get_references(test)
+        rv = self.executor.protocol.marionette._send_message("reftest:run",
+                                                             {"test": self.executor.test_url(test),
+                                                              "references": references,
+                                                              "expected": test.expected(),
+                                                              "timeout": test.timeout * 1000})["value"]
+        return rv
+
+    def get_references(self, node):
+        rv = []
+        for item, relation in node.references:
+            rv.append([self.executor.test_url(item), self.get_references(item), relation])
+        return rv
+
+    def teardown(self):
+        try:
+            self.executor.protocol.marionette._send_message("reftest:teardown", {})
+            self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
+        except socket.error:
+            pass
+
 class WdspecRun(object):
     def __init__(self, func, session, path, timeout):
         self.func = func
         self.result = (None, None)
         self.session = session
         self.path = path
         self.timeout = timeout
         self.result_flag = threading.Event()
@@ -574,17 +634,17 @@ class WdspecRun(object):
             self.result = False, ("ERROR", message)
         finally:
             self.result_flag.set()
 
 
 class MarionetteWdspecExecutor(WdspecExecutor):
     def __init__(self, browser, server_config, webdriver_binary,
                  timeout_multiplier=1, close_after_done=True, debug_info=None,
-                 capabilities=None, webdriver_args=None, binary=None):
+                 capabilities=None, webdriver_args=None, binary=None, **kwargs):
         self.do_delayed_imports()
         WdspecExecutor.__init__(self, browser, server_config,
                                 timeout_multiplier=timeout_multiplier,
                                 debug_info=debug_info)
         self.webdriver_binary = webdriver_binary
         self.webdriver_args = webdriver_args + ["--binary", binary]
         self.capabilities = capabilities
         self.protocol = RemoteMarionetteProtocol(self, browser)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -179,16 +179,23 @@ scheme host and port.""")
                              help="Path to directory containing assertion stack fixing scripts")
     gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
                              default=[], metavar="PREF=VALUE",
                              help="Defines an extra user preference (overrides those in prefs_root)")
     gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
                              help="Enable leak checking")
     gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
                              help="Number of parallel threads to use for stylo")
+    gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
+                             default=None, help="Enable reftest runner implemented inside Marionette")
+    gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
+                             help="Disable reftest runner implemented inside Marionette")
+    gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
+                             choices=["always", "fail", "unexpected"], default="unexpected",
+                             help="With --reftest-internal, when to take a screenshot")
 
     servo_group = parser.add_argument_group("Servo-specific")
     servo_group.add_argument("--user-stylesheet",
                              default=[], action="append", dest="user_stylesheets",
                              help="Inject a user CSS stylesheet into every test.")
 
     sauce_group = parser.add_argument_group("Sauce Labs-specific")
     sauce_group.add_argument("--sauce-browser", dest="sauce_browser",