Bug 1348833 - Bound wpt's --repeat-until-unexpected r?jgraham
Before, we would always repeat indefinitely if this was set. Now,
we only repeat the number of times --repeat is specified. However,
if it is unset, we will still repeat indefinitely until we have an
unexpected result.
MozReview-Commit-ID: GbSNixg7KNL
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -86,17 +86,17 @@ scheme host and port.""")
help="Path to manifest listing tests to include")
test_selection_group.add_argument("--tag", action="append", dest="tags",
help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
- debugging_group.add_argument("--repeat", action="store", type=int, default=1,
+ debugging_group.add_argument("--repeat", action="store", type=int, default=None,
help="Number of times to run the tests")
debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
@@ -343,16 +343,19 @@ def check_args(kwargs):
if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox":
path = exe_path(kwargs["certutil_binary"])
if path is None:
print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
sys.exit(1)
kwargs["certutil_binary"] = path
+ if not kwargs["repeat_until_unexpected"] and kwargs["repeat"] is None:
+ kwargs["repeat"] = 1
+
return kwargs
def check_args_update(kwargs):
set_from_config(kwargs)
if kwargs["product"] is None:
kwargs["product"] = "firefox"
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -161,17 +161,21 @@ def run_tests(config, test_paths, produc
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
- while repeat_count < repeat or repeat_until_unexpected:
+ # If repeat is None, that implies repeat_until_unexpected is
+ # true, and we should repeat indefinitely until an unexpected
+ # result is found. Otherwise just repeat the test the specified
+ # number of times
+ while repeat is None or repeat_count < repeat:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
elif repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)