Bug 1352463 - Define WPT timeouts on the test type level; r?jgraham draft
authorAndreas Tolfsen <ato@mozilla.com>
Fri, 31 Mar 2017 17:38:35 +0100
changeset 564422 c0020f7efea0c81648ad346eaade0c2fb0bd756c
parent 564098 bb38d935d699e0529f9e0bb35578d381026415c4
child 564423 313b46b80ed1373aee2a59c09d0c1c561ef79363
push id54589
push userbmo:ato@mozilla.com
push dateTue, 18 Apr 2017 14:29:38 +0000
reviewersjgraham
bugs1352463
milestone55.0a1
Bug 1352463 - Define WPT timeouts on the test type level; r?jgraham Certain test types have a need for other defaults than the wpttest.DEFAULT_TIMEOUT and wpttest.LONG_TIMEOUT values. This patch changes wptrunner to define default- and long timeouts on a test type level. This allows a test type to override the default durations defined in the abstract Test.default_timeout and Test.long_timeout. Concrete classes, such as ReftestTest and WdspecTest, may override these class properties. MozReview-Commit-ID: IS6df5vuIDC
testing/web-platform/harness/wptrunner/testloader.py
testing/web-platform/harness/wptrunner/wpttest.py
--- a/testing/web-platform/harness/wptrunner/testloader.py
+++ b/testing/web-platform/harness/wptrunner/testloader.py
@@ -87,18 +87,18 @@ class EqualTimeChunker(TestChunker):
 
         for i, (test_type, test_path, tests) in enumerate(manifest_items):
             test_dir = tuple(os.path.split(test_path)[0].split(os.path.sep)[:3])
 
             if not test_dir in by_dir:
                 by_dir[test_dir] = PathData(test_dir)
 
             data = by_dir[test_dir]
-            time = sum(wpttest.DEFAULT_TIMEOUT if test.timeout !=
-                       "long" else wpttest.LONG_TIMEOUT for test in tests)
+            time = sum(test.default_timeout if test.timeout !=
+                       "long" else test.long_timeout for test in tests)
             data.time += time
             total_time += time
             data.tests.append((test_type, test_path, tests))
 
         return by_dir, total_time
 
     def _maybe_remove(self, chunks, i, direction):
         """Trial removing a chunk from one chunk to an adjacent one.
--- a/testing/web-platform/harness/wptrunner/wpttest.py
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -1,15 +1,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-DEFAULT_TIMEOUT = 10  # seconds
-LONG_TIMEOUT = 60  # seconds
-
 import os
 
 import mozinfo
 
 from wptmanifest.parser import atoms
 
 atom_reset = atoms["Reset"]
 enabled_tests = set(["testharness", "reftest", "wdspec"])
@@ -94,36 +91,40 @@ class RunInfo(dict):
                 break
             dirs.add(str(path))
             path = os.path.split(path)[0]
 
         mozinfo.find_and_update_from_json(*dirs)
 
 
 class Test(object):
+
     result_cls = None
     subtest_result_cls = None
     test_type = None
 
+    default_timeout = 10  # seconds
+    long_timeout = 60  # seconds
+
     def __init__(self, tests_root, url, inherit_metadata, test_metadata,
-                 timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
+                 timeout=None, path=None, protocol="http"):
         self.tests_root = tests_root
         self.url = url
         self._inherit_metadata = inherit_metadata
         self._test_metadata = test_metadata
-        self.timeout = timeout
+        self.timeout = timeout if timeout is not None else self.default_timeout
         self.path = path
         self.environment = {"protocol": protocol, "prefs": self.prefs}
 
     def __eq__(self, other):
         return self.id == other.id
 
     @classmethod
     def from_manifest(cls, manifest_item, inherit_metadata, test_metadata):
-        timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
+        timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout
         protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http"
         return cls(manifest_item.source_file.tests_root,
                    manifest_item.url,
                    inherit_metadata,
                    test_metadata,
                    timeout=timeout,
                    path=manifest_item.source_file.path,
                    protocol=protocol)
@@ -244,18 +245,17 @@ class ManualTest(Test):
         return self.url
 
 
 class ReftestTest(Test):
     result_cls = ReftestResult
     test_type = "reftest"
 
     def __init__(self, tests_root, url, inherit_metadata, test_metadata, references,
-                 timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
-                 dpi=None, protocol="http"):
+                 timeout=None, path=None, viewport_size=None, dpi=None, protocol="http"):
         Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
                       path, protocol)
 
         for _, ref_type in references:
             if ref_type not in ("==", "!="):
                 raise ValueError
 
         self.references = references
@@ -265,17 +265,17 @@ class ReftestTest(Test):
     @classmethod
     def from_manifest(cls,
                       manifest_test,
                       inherit_metadata,
                       test_metadata,
                       nodes=None,
                       references_seen=None):
 
-        timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT
+        timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout
 
         if nodes is None:
             nodes = {}
         if references_seen is None:
             references_seen = set()
 
         url = manifest_test.url
 
@@ -329,20 +329,24 @@ class ReftestTest(Test):
         return self.url
 
     @property
     def keys(self):
         return ("reftype", "refurl")
 
 
 class WdspecTest(Test):
+
     result_cls = WdspecResult
     subtest_result_cls = WdspecSubtestResult
     test_type = "wdspec"
 
+    default_timeout = 10
+    long_timeout = 60
+
 
 manifest_test_cls = {"reftest": ReftestTest,
                      "testharness": TestharnessTest,
                      "manual": ManualTest,
                      "wdspec": WdspecTest}
 
 
 def from_manifest(manifest_test, inherit_metadata, test_metadata):