Bug 1344991 - Continue reftest after a crash r=dbaron,jmaher
MozReview-Commit-ID: CTUaweql66d
--- a/layout/tools/reftest/reftest.jsm
+++ b/layout/tools/reftest/reftest.jsm
@@ -120,16 +120,18 @@ var gUnexpectedCrashDumpFiles = { };
var gCrashDumpDir;
var gPendinCrashDumpDir;
var gFailedNoPaint = false;
var gFailedOpaqueLayer = false;
var gFailedOpaqueLayerMessages = [];
var gFailedAssignedLayer = false;
var gFailedAssignedLayerMessages = [];
+var gStartAfter = undefined;
+
// The enabled-state of the test-plugins, stored so they can be reset later
var gTestPluginEnabledStates = null;
const TYPE_REFTEST_EQUAL = '==';
const TYPE_REFTEST_NOTEQUAL = '!=';
const TYPE_LOAD = 'load'; // test without a reference (just test that it does
// not assert, crash, hang, or leak)
const TYPE_SCRIPT = 'script'; // test contains individual test results
@@ -384,16 +386,22 @@ function InitAndStartRefTests()
gTotalChunks = 0;
gThisChunk = 0;
}
try {
gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode");
} catch(e) {}
+ try {
+ gStartAfter = prefs.getCharPref("reftest.startAfter");
+ } catch(e) {
+ gStartAfter = undefined;
+ }
+
#ifdef MOZ_STYLO
try {
gCompareStyloToGecko = prefs.getBoolPref("reftest.compareStyloToGecko");
} catch(e) {}
#endif
gWindowUtils = gContainingWindow.QueryInterface(CI.nsIInterfaceRequestor).getInterface(CI.nsIDOMWindowUtils);
if (!gWindowUtils || !gWindowUtils.compareCanvases)
@@ -539,17 +547,34 @@ function StartTests()
end = gThisChunk == gTotalChunks ? gURLs.length : gURLs.indexOf(tURLs[end + 1]) - 1;
gURLs = gURLs.slice(start, end);
logger.info("Running chunk " + gThisChunk + " out of " + gTotalChunks + " chunks. " +
"tests " + (start+1) + "-" + end + "/" + gURLs.length);
}
if (gShuffle) {
+ if (gStartAfter !== undefined) {
+ logger.error("Can't resume from a crashed test when " +
+ "--shuffle is enabled, continue by shuffling " +
+ "all the tests");
+ DoneTests();
+ return;
+ }
Shuffle(gURLs);
+ } else if (gStartAfter !== undefined) {
+ // Skip through previously crashed test
+ // We have to do this after chunking so we don't break the numbers
+ var crash_idx = gURLs.map(function(url) {
+ return url['url1']['spec'];
+ }).indexOf(gStartAfter);
+ if (crash_idx == -1) {
+ throw "Can't find the previously crashed test";
+ }
+ gURLs = gURLs.slice(crash_idx + 1);
}
gTotalTests = gURLs.length;
if (!gTotalTests)
throw "No tests to run";
gURICanvases = {};
--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -231,22 +231,25 @@ class RemoteReftest(RefTest):
options.xrePath = remoteXrePath
options.utilityPath = remoteUtilityPath
return 0
def stopWebServer(self, options):
self.server.stop()
- def createReftestProfile(self, options, manifest):
+ def createReftestProfile(self, options, manifest, startAfter=None):
profile = RefTest.createReftestProfile(self,
options,
manifest,
server=options.remoteWebServer,
port=options.httpPort)
+ if startAfter is not None:
+ print ("WARNING: Continuing after a crash is not supported for remote "
+ "reftest yet.")
profileDir = profile.profile
prefs = {}
prefs["app.update.url.android"] = ""
prefs["browser.firstrun.show.localepicker"] = False
prefs["reftest.remote"] = True
prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
@@ -313,17 +316,17 @@ class RemoteReftest(RefTest):
binary,
profile.profile,
cmdargs,
utilityPath=options.utilityPath,
xrePath=options.xrePath,
debuggerInfo=debuggerInfo,
symbolsPath=symbolsPath,
timeout=timeout)
- return status
+ return status, None
def cleanup(self, profileDir):
# Pull results back from device
if self.remoteLogFile and \
self._devicemanager.fileExists(self.remoteLogFile):
self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
else:
print "WARNING: Unable to retrieve log file (%s) from remote " \
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -242,17 +242,17 @@ class RefTest(object):
path = os.path.split(path)[0]
mozinfo.find_and_update_from_json(*dirs)
def getFullPath(self, path):
"Get an absolute path relative to self.oldcwd."
return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
def createReftestProfile(self, options, manifests, server='localhost', port=0,
- profile_to_clone=None):
+ profile_to_clone=None, startAfter=None):
"""Sets up a profile for reftest.
:param options: Object containing command line options
:param manifests: Dictionary of the form {manifest_path: [filters]}
:param server: Server name to use for http tests
:param profile_to_clone: Path to a profile to use as the basis for the
test profile
"""
@@ -281,16 +281,19 @@ class RefTest(object):
if options.runUntilFailure:
prefs['reftest.runUntilFailure'] = True
if options.cleanupCrashes:
prefs['reftest.cleanupPendingCrashes'] = True
prefs['reftest.focusFilterMode'] = options.focusFilterMode
prefs['reftest.logLevel'] = options.log_tbpl_level or 'info'
prefs['reftest.manifests'] = json.dumps(manifests)
+ if startAfter is not None:
+ prefs['reftest.startAfter'] = startAfter
+
if options.e10s:
prefs['browser.tabs.remote.autostart'] = True
prefs['extensions.e10sBlocksEnabling'] = False
# Bug 1262954: For winXP + e10s disable acceleration
if platform.system() in ("Windows", "Microsoft") and \
'5.1' in platform.version() and options.e10s:
prefs['layers.acceleration.disabled'] = True
@@ -344,17 +347,18 @@ class RefTest(object):
kwargs = {'addons': addons,
'preferences': prefs,
'locations': locations}
if profile_to_clone:
profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
else:
profile = mozprofile.Profile(**kwargs)
- options.extraProfileFiles.append(os.path.join(here, 'chrome'))
+ if os.path.join(here, 'chrome') not in options.extraProfileFiles:
+ options.extraProfileFiles.append(os.path.join(here, 'chrome'))
self.copyExtraFilesToProfile(options, profile)
return profile
def environment(self, **kwargs):
kwargs['log'] = self.log
return test_environment(**kwargs)
@@ -651,56 +655,75 @@ class RefTest(object):
self.lastTestSeen = 'Main app process exited normally'
crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'),
symbolsPath, test=self.lastTestSeen)
runner.cleanup()
if not status and crashed:
status = 1
- return status
+ return status, self.lastTestSeen
def runSerialTests(self, manifests, options, cmdargs=None):
debuggerInfo = None
if options.debugger:
debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
options.debuggerInteractive)
profileDir = None
- try:
- if cmdargs is None:
- cmdargs = []
+ startAfter = None # When the previous run crashed, we skip the tests we ran before
+ prevStartAfter = None
+ status = 1 # Just to start the loop
+ while status != 0:
+ try:
+ if cmdargs is None:
+ cmdargs = []
- if self.use_marionette:
- cmdargs.append('-marionette')
+ if self.use_marionette:
+ cmdargs.append('-marionette')
- profile = self.createReftestProfile(options, manifests)
- profileDir = profile.profile # name makes more sense
+ profile = self.createReftestProfile(options,
+ manifests,
+ startAfter=startAfter)
+ profileDir = profile.profile # name makes more sense
- # browser environment
- browserEnv = self.buildBrowserEnv(options, profileDir)
+ # browser environment
+ browserEnv = self.buildBrowserEnv(options, profileDir)
- self.log.info("Running with e10s: {}".format(options.e10s))
- status = self.runApp(profile,
- binary=options.app,
- cmdargs=cmdargs,
- # give the JS harness 30 seconds to deal with
- # its own timeouts
- env=browserEnv,
- timeout=options.timeout + 30.0,
- symbolsPath=options.symbolsPath,
- options=options,
- debuggerInfo=debuggerInfo)
- self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
- mozleak.process_leak_log(self.leakLogFile,
- leak_thresholds=options.leakThresholds,
- stack_fixer=get_stack_fixer_function(options.utilityPath,
- options.symbolsPath))
- finally:
- self.cleanup(profileDir)
+ self.log.info("Running with e10s: {}".format(options.e10s))
+ status, startAfter = self.runApp(profile,
+ binary=options.app,
+ cmdargs=cmdargs,
+ # give the JS harness 30 seconds to deal with
+ # its own timeouts
+ env=browserEnv,
+ timeout=options.timeout + 30.0,
+ symbolsPath=options.symbolsPath,
+ options=options,
+ debuggerInfo=debuggerInfo)
+ self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
+ mozleak.process_leak_log(self.leakLogFile,
+ leak_thresholds=options.leakThresholds,
+ stack_fixer=get_stack_fixer_function(options.utilityPath,
+ options.symbolsPath))
+ self.cleanup(profileDir)
+ if startAfter is not None and options.shuffle:
+ self.log.error("Can not resume from a crash with --shuffle "
+ "enabled. Please consider disabling --shuffle")
+ break
+ if startAfter == prevStartAfter:
+ # If the test stuck on the same test, or there the crashed
+ # test appeared more then once, stop
+ self.log.error("Force stop because we keep running into "
+ "test \"{}\"".format(startAfter))
+ break
+ prevStartAfter = startAfter
+ # TODO: we need to emit an SUITE-END log if it crashed
+ finally:
+ self.cleanup(profileDir)
return status
def copyExtraFilesToProfile(self, options, profile):
"Copy extra files or dirs specified on the command line to the testing profile."
profileDir = profile.profile
if not os.path.exists(os.path.join(profileDir, "hyphenation")):
os.makedirs(os.path.join(profileDir, "hyphenation"))
for f in options.extraProfileFiles: