Bug 1383922 - Allow try-test-paths syntax to refer to wpt subtypes draft
authorMaja Frydrychowicz <mjzffr@gmail.com>
Wed, 26 Jul 2017 19:40:59 -0400
changeset 618586 f51cc78e8ba2b1a39dc8f948addf313a49a64d3d
parent 618383 26516ba270816a6cc90f5c42a9b66701369a551f
child 640124 df8f9971a8b24a043fbec47a68cf4361574ddff5
push id71392
push userbmo:mjzffr@gmail.com
push dateMon, 31 Jul 2017 19:25:05 +0000
bugs1383922
milestone56.0a1
Bug 1383922 - Allow try-test-paths syntax to refer to wpt subtypes This allows syntax like --try-test-paths web-platform-tests-reftests:path/to/test MozReview-Commit-ID: uAet1ilPVy
testing/mozharness/mozharness/mozilla/testing/try_tools.py
testing/mozharness/scripts/web_platform_tests.py
--- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py
+++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py
@@ -31,17 +31,23 @@ test_flavors = {
     'reftest': {
         "path": lambda x: os.path.join("tests", "reftest", "tests", x)
     },
     'crashtest': {
         "path": lambda x: os.path.join("tests", "reftest", "tests", x)
     },
     'web-platform-tests': {
         "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
-    }
+    },
+    'web-platform-tests-reftests': {
+        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
+    },
+    'web-platform-tests-wdspec': {
+        "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
+    },
 }
 
 class TryToolsMixin(TransferMixin):
     """Utility functions for an interface between try syntax and out test harnesses.
     Requires log and script mixins."""
 
     harness_extra_args = None
     try_test_paths = {}
@@ -241,17 +247,17 @@ class TryToolsMixin(TransferMixin):
             args = self.harness_extra_args.get(flavor, [])[:]
 
         if self.try_test_paths.get(flavor):
             self.info('TinderboxPrint: Tests will be run from the following '
                       'files: %s.' % ','.join(self.try_test_paths[flavor]))
             args.extend(['--this-chunk=1', '--total-chunks=1'])
 
             path_func = test_flavors[flavor].get("path", lambda x:x)
-            tests = [path_func(item) for item in self.try_test_paths[flavor]]
+            tests = [path_func(os.path.normpath(item)) for item in self.try_test_paths[flavor]]
         else:
             tests = []
 
         if args or tests:
             self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
                       'to the test command:\nTinderboxPrint: \t%s -- %s' %
                       (" ".join(args), " ".join(tests)))
 
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -168,59 +168,62 @@ class WebPlatformTest(TestingMixin, Merc
                 "--symbols-path=%s" % self.query_symbols_url(),
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
                 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"),
                 "--run-by-dir=3"]
 
         if not sys.platform.startswith("linux"):
             cmd += ["--exclude=css"]
 
-        # Let wptrunner determine the test type when --try-test-paths is used
-        wpt_test_paths = self.try_test_paths.get("web-platform-tests")
-        if not wpt_test_paths:
-            for test_type in c.get("test_type", []):
-                cmd.append("--test-type=%s" % test_type)
+        for test_type in c.get("test_type", []):
+            cmd.append("--test-type=%s" % test_type)
 
         if not c["e10s"]:
             cmd.append("--disable-e10s")
 
         if c["single_stylo_traversal"]:
             cmd.append("--stylo-threads=1")
         else:
             cmd.append("--stylo-threads=4")
 
         for opt in ["total_chunks", "this_chunk"]:
             val = c.get(opt)
             if val:
                 cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 
-        if wpt_test_paths or "wdspec" in c.get("test_type", []):
+        if "wdspec" in c.get("test_type", []):
             geckodriver_path = os.path.join(dirs["abs_test_bin_dir"], "geckodriver")
             if not os.path.isfile(geckodriver_path):
                 self.fatal("Unable to find geckodriver binary "
                            "in common test package: %s" % geckodriver_path)
             cmd.append("--webdriver-binary=%s" % geckodriver_path)
 
         options = list(c.get("options", []))
 
         str_format_values = {
             'binary_path': self.binary_path,
             'test_path': dirs["abs_wpttest_dir"],
             'test_install_path': dirs["abs_test_install_dir"],
             'abs_app_dir': abs_app_dir,
             'abs_work_dir': dirs["abs_work_dir"]
             }
 
-        try_options, try_tests = self.try_args("web-platform-tests")
+        test_type_suite = {
+            "testharness": "web-platform-tests",
+            "reftest": "web-platform-tests-reftests",
+            "wdspec": "web-platform-tests-wdspec",
+        }
+        for test_type in c.get("test_type", []):
+            try_options, try_tests = self.try_args(test_type_suite[test_type])
 
-        cmd.extend(self.query_options(options,
-                                      try_options,
-                                      str_format_values=str_format_values))
-        cmd.extend(self.query_tests_args(try_tests,
-                                         str_format_values=str_format_values))
+            cmd.extend(self.query_options(options,
+                                          try_options,
+                                          str_format_values=str_format_values))
+            cmd.extend(self.query_tests_args(try_tests,
+                                             str_format_values=str_format_values))
 
         return cmd
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
             extract_dirs=["mach",
                           "bin/*",
                           "config/*",