Bug 1372981 - Limit reftest to 4 attempts at recovering after a crash, r?shinglyu draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 14 Jun 2017 13:25:59 -0400
changeset 594221 8e6a72be905cc9893c76e73b391e142ca715e060
parent 594056 ad3f1138ce6f199408ad58d65c7476636e924909
child 633353 6cc80968d456f754d1f087adb969f26bbbd7abad
push id63954
push userahalberstadt@mozilla.com
push dateWed, 14 Jun 2017 17:48:41 +0000
reviewersshinglyu
bugs1372981
milestone56.0a1
Bug 1372981 - Limit reftest to 4 attempts at recovering after a crash, r?shinglyu MozReview-Commit-ID: IEfM1b6FTHD
layout/tools/reftest/reftestcommandline.py
layout/tools/reftest/runreftest.py
--- a/layout/tools/reftest/reftestcommandline.py
+++ b/layout/tools/reftest/reftestcommandline.py
@@ -224,16 +224,23 @@ class ReftestArgumentsParser(argparse.Ar
                           help=argparse.SUPPRESS)
 
         self.add_argument("--cleanup-crashes",
                           action="store_true",
                           dest="cleanupCrashes",
                           default=False,
                           help="Delete pending crash reports before running tests.")
 
+        self.add_argument("--max-retries",
+                          type=int,
+                          dest="maxRetries",
+                          default=4,
+                          help="The maximum number of attempts to try and recover from a "
+                               "crash before aborting the test run [default 4].")
+
         self.add_argument("tests",
                           metavar="TEST_PATH",
                           nargs="*",
                           help="Path to test file, manifest file, or directory containing tests")
 
         mozlog.commandline.add_logging_group(self)
 
     def get_ip(self):
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
 Runs the reftest test harness.
 """
 
 import collections
+import itertools
 import json
 import multiprocessing
 import os
 import platform
 import re
 import shutil
 import signal
 import subprocess
@@ -665,18 +666,17 @@ class RefTest(object):
         debuggerInfo = None
         if options.debugger:
             debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
                                                       options.debuggerInteractive)
 
         profileDir = None
         startAfter = None  # When the previous run crashed, we skip the tests we ran before
         prevStartAfter = None
-        status = 1  # Just to start the loop
-        while status != 0:
+        for i in itertools.count():
             try:
                 if cmdargs is None:
                     cmdargs = []
 
                 if self.use_marionette:
                     cmdargs.append('-marionette')
 
                 profile = self.createReftestProfile(options,
@@ -698,21 +698,27 @@ class RefTest(object):
                                                  symbolsPath=options.symbolsPath,
                                                  options=options,
                                                  debuggerInfo=debuggerInfo)
                 self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
                 mozleak.process_leak_log(self.leakLogFile,
                                          leak_thresholds=options.leakThresholds,
                                          stack_fixer=get_stack_fixer_function(options.utilityPath,
                                                                               options.symbolsPath))
-                self.cleanup(profileDir)
+                if status == 0:
+                    break
+
                 if startAfter is not None and options.shuffle:
                     self.log.error("Can not resume from a crash with --shuffle "
                                    "enabled. Please consider disabling --shuffle")
                     break
+                if startAfter is not None and options.maxRetries <= i:
+                    self.log.error("Hit maximum number of allowed retries ({}) "
+                                   "in the test run".format(options.maxRetries))
+                    break
                 if startAfter == prevStartAfter:
                     # If the test stuck on the same test, or there the crashed
                     # test appeared more then once, stop
                     self.log.error("Force stop because we keep running into "
                                    "test \"{}\"".format(startAfter))
                     break
                 prevStartAfter = startAfter
                 # TODO: we need to emit an SUITE-END log if it crashed