Bug 1294820 - Add support for running wpt from one-click loaner mach environment, r=ahal draft
authorJames Graham <james@hoppipolla.co.uk>
Mon, 19 Jun 2017 19:20:41 +0100
changeset 597597 0af1c10df823237879baaa7b1dafd0c4508e3832
parent 597455 464b2a3c25aa1065760d9ecbb0870bca4a66c62e
child 634267 32c83870fe2f608d59e20a63d0134bc9d0761a8f
push id64972
push userbmo:james@hoppipolla.co.uk
push dateTue, 20 Jun 2017 17:47:19 +0000
reviewersahal
bugs1294820
milestone56.0a1
Bug 1294820 - Add support for running wpt from one-click loaner mach environment, r=ahal Allow running |mach wpt| on one click loaners in order to run web-platform-tests tests. This implementation is just like the one for other testsuites using thee packaged tests rather than the checkout that we get with wpt, at least on Linux. That's also where the tests run from so it seems reasonable for now. Moving to the checkout in the future could remove some of the logic here by using a fake mozbuild environment so that the testsuite itself doesn't have to implement anything much. MozReview-Commit-ID: CaewrdjJ2ef
testing/mozharness/scripts/web_platform_tests.py
testing/tools/mach_test_package_bootstrap.py
testing/web-platform/mach_commands.py
testing/web-platform/mach_commands_base.py
testing/web-platform/mach_test_package_commands.py
testing/web-platform/moz.build
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -202,21 +202,22 @@ class WebPlatformTest(TestingMixin, Merc
                                       str_format_values=str_format_values))
         cmd.extend(self.query_tests_args(try_tests,
                                          str_format_values=str_format_values))
 
         return cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
-            extract_dirs=["bin/*",
+            extract_dirs=["mach",
+                          "bin/*",
                           "config/*",
                           "mozbase/*",
                           "marionette/*",
-                          "tools/wptserve/*",
+                          "tools/*",
                           "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):
         dirs = self.query_abs_dirs()
         cmd = self._query_cmd()
 
         parser = StructuredOutputParser(config=self.config,
--- a/testing/tools/mach_test_package_bootstrap.py
+++ b/testing/tools/mach_test_package_bootstrap.py
@@ -32,25 +32,28 @@ SEARCH_PATHS = [
     'mozbase/mozrunner',
     'mozbase/mozscreenshot',
     'mozbase/mozsystemmonitor',
     'mozbase/moztest',
     'mozbase/mozversion',
     'reftest',
     'tools/mach',
     'tools/wptserve',
+    'web-platform',
+    'web-platform/tests/tools/wptrunner',
     'xpcshell',
 ]
 
 # Individual files providing mach commands.
 MACH_MODULES = [
     'marionette/mach_test_package_commands.py',
     'mochitest/mach_test_package_commands.py',
     'reftest/mach_test_package_commands.py',
     'tools/mach/mach/commands/commandinfo.py',
+    'web-platform/mach_test_package_commands.py',
     'xpcshell/mach_test_package_commands.py',
 ]
 
 
 CATEGORIES = {
     'testing': {
         'short': 'Testing',
         'long': 'Run tests.',
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -15,25 +15,23 @@ from mozbuild.base import (
     MozbuildObject,
 )
 
 from mach.decorators import (
     CommandProvider,
     Command,
 )
 
-# This should probably be consolidated with similar classes in other test
-# runners.
-class InvalidTestPathError(Exception):
-    """Exception raised when the test path is not valid."""
+from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
+
 
-class WebPlatformTestsRunner(MozbuildObject):
-    """Run web platform tests."""
+class WebPlatformTestsRunnerSetup(MozbuildObject):
+    default_log_type = "mach"
 
-    def setup_kwargs_firefox(self, kwargs):
+    def kwargs_firefox(self, kwargs):
         from wptrunner import wptcommandline
 
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
             sys.path.append(build_path)
 
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
@@ -65,17 +63,17 @@ class WebPlatformTestsRunner(MozbuildObj
 
         kwargs["capture_stdio"] = True
 
         if kwargs["webdriver_binary"] is None:
             kwargs["webdriver_binary"] = self.get_binary_path("geckodriver", validate_exists=False)
 
         kwargs = wptcommandline.check_args(kwargs)
 
-    def setup_kwargs_wptrun(self, kwargs):
+    def kwargs_wptrun(self, kwargs):
         from wptrunner import wptcommandline
         here = os.path.join(self.topsrcdir, 'testing', 'web-platform')
 
         sys.path.insert(0, os.path.join(here, "tests", "tools"))
 
         import wptrun
 
         product = kwargs["product"]
@@ -108,37 +106,16 @@ class WebPlatformTestsRunner(MozbuildObj
         dest_manifest = os.path.join(kwargs["metadata_root"], "MANIFEST.json")
 
         if not os.path.exists(dest_manifest) and os.path.exists(src_manifest):
             with open(src_manifest) as src, open(dest_manifest, "w") as dest:
                 dest.write(src.read())
 
         kwargs = wptcommandline.check_args(kwargs)
 
-    def run_tests(self, **kwargs):
-        from wptrunner import wptrunner
-
-        if kwargs["product"] in ["firefox", None]:
-            self.setup_kwargs_firefox(kwargs)
-        elif kwargs["product"] in ("chrome", "edge", "servo"):
-            self.setup_kwargs_wptrun(kwargs)
-        else:
-            raise ValueError("Unknown product %s" % kwargs["product"])
-
-        logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
-        result = wptrunner.run_tests(**kwargs)
-
-        return int(not result)
-
-    def list_test_groups(self, **kwargs):
-        from wptrunner import wptrunner
-
-        self.setup_kwargs(kwargs)
-
-        wptrunner.list_test_groups(**kwargs)
 
 class WebPlatformTestsUpdater(MozbuildObject):
     """Update web platform tests."""
     def run_update(self, **kwargs):
         import update
         from update import updatecommandline
 
         if kwargs["config"] is None:
@@ -302,20 +279,16 @@ class WPTManifestUpdater(MozbuildObject)
     def run_update(self, check_clean=False, rebuild=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
 
 
-def create_parser_wpt():
-    from wptrunner import wptcommandline
-    return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
-
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
 def create_parser_reduce():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser_reduce()
 
@@ -359,22 +332,19 @@ class MachCommands(MachCommandBase):
     def run_web_platform_tests(self, **params):
         self.setup()
 
         if "test_objects" in params:
             for item in params["test_objects"]:
                 params["include"].append(item["name"])
             del params["test_objects"]
 
-        wpt_runner = self._spawn(WebPlatformTestsRunner)
-
-        if params["list_test_groups"]:
-            return wpt_runner.list_test_groups(**params)
-        else:
-            return wpt_runner.run_tests(**params)
+        wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
+        wpt_runner = WebPlatformTestsRunner(wpt_setup)
+        return wpt_runner.run(**params)
 
     @Command("wpt",
              category="testing",
              conditions=[conditions.is_firefox],
              parser=create_parser_wpt)
     def run_wpt(self, **params):
         return self.run_web_platform_tests(**params)
 
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mach_commands_base.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+
+def create_parser_wpt():
+    from wptrunner import wptcommandline
+    return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
+
+
+class WebPlatformTestsRunner(object):
+    """Run web platform tests."""
+
+    def __init__(self, setup):
+        self.setup = setup
+
+    def run(self, **kwargs):
+        from wptrunner import wptrunner
+        if kwargs["product"] in ["firefox", None]:
+            self.setup.kwargs_firefox(kwargs)
+        elif kwargs["product"] in ("chrome", "edge", "servo"):
+            self.setup.kwargs_wptrun(kwargs)
+        else:
+            raise ValueError("Unknown product %s" % kwargs["product"])
+        logger = wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout})
+        result = wptrunner.start(**kwargs)
+        return int(not result)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/mach_test_package_commands.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
+from mach.decorators import (
+    CommandProvider,
+    Command,
+)
+
+
+class WebPlatformTestsRunnerSetup(object):
+    default_log_type = "tbpl"
+
+    def __init__(self, context):
+        self.context = context
+
+    def kwargs_firefox(self, kwargs):
+        from wptrunner import wptcommandline
+        if kwargs["config"] is None:
+            kwargs["config"] = os.path.join(self.context.package_root, 'web-platform', 'wptrunner.ini')
+        if kwargs["binary"] is None:
+            kwargs["binary"] = self.context.firefox_bin
+        if kwargs["prefs_root"] is None:
+            kwargs["prefs_root"] = os.path.join(self.context.package_root, 'web-platform', "prefs")
+        if kwargs["certutil_binary"] is None:
+            kwargs["certutil_binary"] = os.path.join(self.context.bin_dir, 'certutil')
+        if kwargs["stackfix_dir"] is None:
+            kwargs["stackfix_dir"] = self.context.bin_dir
+        if kwargs["ssl_type"] in (None, "pregenerated"):
+            if kwargs["ca_cert_path"] is None:
+                kwargs["ca_cert_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "cacert.pem")
+            if kwargs["host_key_path"] is None:
+                kwargs["host_key_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "web-platform.test.key")
+            if kwargs["host_cert_path"] is None:
+                kwargs["host_cert_path"] = os.path.join(self.context.package_root, "web-platform", "certs", "web-platform.test.pem")
+        kwargs["capture_stdio"] = True
+
+        if kwargs["webdriver_binary"] is None:
+            kwargs["webdriver_binary"] = os.path.join(self.context.bin_dir, "geckodriver")
+
+        kwargs = wptcommandline.check_args(kwargs)
+
+    def kwargs_wptrun(self, kwargs):
+        raise NotImplementedError
+
+
+@CommandProvider
+class MachCommands(object):
+    def __init__(self, context):
+        self.context = context
+
+    @Command("web-platform-tests",
+             category="testing",
+             parser=create_parser_wpt)
+    def run_web_platform_tests(self, **kwargs):
+        self.context.activate_mozharness_venv()
+        return WebPlatformTestsRunner(WebPlatformTestsRunnerSetup(self.context)).run(**kwargs)
+
+    @Command("wpt",
+             category="testing",
+             parser=create_parser_wpt)
+    def run_wpt(self, **params):
+        return self.run_web_platform_tests(**params)
--- a/testing/web-platform/moz.build
+++ b/testing/web-platform/moz.build
@@ -5,16 +5,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 WEB_PLATFORM_TESTS_MANIFESTS += [
     ('meta/MANIFEST.json', 'tests/'),
     ('mozilla/meta/MANIFEST.json', 'mozilla/tests/')
 ]
 
 TEST_HARNESS_FILES['web-platform'] += [
+    'mach_commands_base.py',
+    'mach_test_package_commands.py',
     'outbound/**',
     'runtests.py',
     'wptrunner.ini'
 ]
 
 TEST_HARNESS_FILES['web-platform'].certs = [
     'certs/cacert.pem',
     'certs/web-platform.test.key',