Bug 1348833 - Bound wpt's --repeat-until-unexpected r?jgraham draft
authorWilliam Lachance <wlachance@mozilla.com>
Tue, 28 Mar 2017 08:08:20 -0400
changeset 552397 a2daa57d8c81895ecb2ad1b462f3b5ac4947492f
parent 551789 9577ddeaafd85554c2a855f385a87472a089d5c0
child 552398 182bb4e0783231f08723e04e6ff7b1d2ce0e4ace
push id51339
push userwlachance@mozilla.com
push dateTue, 28 Mar 2017 12:11:54 +0000
reviewersjgraham
bugs1348833
milestone55.0a1
Bug 1348833 - Bound wpt's --repeat-until-unexpected r?jgraham Before, we would always repeat indefinitely if this was set. Now, we only repeat the number of times --repeat is specified. However, if it is unset, we will still repeat indefinitely until we have an unexpected result. MozReview-Commit-ID: GbSNixg7KNL
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -86,17 +86,17 @@ scheme host and port.""")
                                       help="Path to manifest listing tests to include")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
 
     debugging_group = parser.add_argument_group("Debugging")
     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
-    debugging_group.add_argument("--repeat", action="store", type=int, default=1,
+    debugging_group.add_argument("--repeat", action="store", type=int, default=None,
                                  help="Number of times to run the tests")
     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
                                  help="Run tests in a loop until one returns an unexpected result")
     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
                                  help="Don't halt the test runner irrespective of the number of tests run")
 
@@ -343,16 +343,19 @@ def check_args(kwargs):
 
     if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox":
         path = exe_path(kwargs["certutil_binary"])
         if path is None:
             print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
             sys.exit(1)
         kwargs["certutil_binary"] = path
 
+    if not kwargs["repeat_until_unexpected"] and kwargs["repeat"] is None:
+        kwargs["repeat"] = 1
+
     return kwargs
 
 
 def check_args_update(kwargs):
     set_from_config(kwargs)
 
     if kwargs["product"] is None:
         kwargs["product"] = "firefox"
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -161,17 +161,21 @@ def run_tests(config, test_paths, produc
                 raise
 
             browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
 
             repeat = kwargs["repeat"]
             repeat_count = 0
             repeat_until_unexpected = kwargs["repeat_until_unexpected"]
 
-            while repeat_count < repeat or repeat_until_unexpected:
+            # If repeat is None, that implies repeat_until_unexpected is
+            # true, and we should repeat indefinitely until an unexpected
+            # result is found. Otherwise just repeat the test the specified
+            # number of times
+            while repeat is None or repeat_count < repeat:
                 repeat_count += 1
                 if repeat_until_unexpected:
                     logger.info("Repetition %i" % (repeat_count))
                 elif repeat > 1:
                     logger.info("Repetition %i / %i" % (repeat_count, repeat))
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)