Bug 1344991 - Continue reftest after a crash r=dbaron,jmaher draft
authorShing Lyu <slyu@mozilla.com>
Thu, 09 Mar 2017 10:49:20 +0800
changeset 564036 a7732415fb0766ab631743aca494c0ab272e87f8
parent 563802 a374c35469935a874fefe64d3e07003fc5bc8884
child 624649 6eb009c34ad2296e931286cf5aa714d78302c448
push id54501
push userbmo:slyu@mozilla.com
push dateTue, 18 Apr 2017 07:53:59 +0000
reviewersdbaron, jmaher
bugs1344991
milestone55.0a1
Bug 1344991 - Continue reftest after a crash r=dbaron,jmaher MozReview-Commit-ID: CTUaweql66d
layout/tools/reftest/reftest.jsm
layout/tools/reftest/remotereftest.py
layout/tools/reftest/runreftest.py
--- a/layout/tools/reftest/reftest.jsm
+++ b/layout/tools/reftest/reftest.jsm
@@ -120,16 +120,18 @@ var gUnexpectedCrashDumpFiles = { };
 var gCrashDumpDir;
 var gPendinCrashDumpDir;
 var gFailedNoPaint = false;
 var gFailedOpaqueLayer = false;
 var gFailedOpaqueLayerMessages = [];
 var gFailedAssignedLayer = false;
 var gFailedAssignedLayerMessages = [];
 
+var gStartAfter = undefined;
+
 // The enabled-state of the test-plugins, stored so they can be reset later
 var gTestPluginEnabledStates = null;
 
 const TYPE_REFTEST_EQUAL = '==';
 const TYPE_REFTEST_NOTEQUAL = '!=';
 const TYPE_LOAD = 'load';     // test without a reference (just test that it does
                               // not assert, crash, hang, or leak)
 const TYPE_SCRIPT = 'script'; // test contains individual test results
@@ -384,16 +386,22 @@ function InitAndStartRefTests()
         gTotalChunks = 0;
         gThisChunk = 0;
     }
 
     try {
         gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode");
     } catch(e) {}
 
+    try {
+        gStartAfter = prefs.getCharPref("reftest.startAfter");
+    } catch(e) {
+        gStartAfter = undefined;
+    }
+
 #ifdef MOZ_STYLO
     try {
         gCompareStyloToGecko = prefs.getBoolPref("reftest.compareStyloToGecko");
     } catch(e) {}
 #endif
 
     gWindowUtils = gContainingWindow.QueryInterface(CI.nsIInterfaceRequestor).getInterface(CI.nsIDOMWindowUtils);
     if (!gWindowUtils || !gWindowUtils.compareCanvases)
@@ -539,17 +547,34 @@ function StartTests()
             end = gThisChunk == gTotalChunks ? gURLs.length : gURLs.indexOf(tURLs[end + 1]) - 1;
             gURLs = gURLs.slice(start, end);
 
             logger.info("Running chunk " + gThisChunk + " out of " + gTotalChunks + " chunks.  " +
                 "tests " + (start+1) + "-" + end + "/" + gURLs.length);
         }
 
         if (gShuffle) {
+            if (gStartAfter !== undefined) {
+                logger.error("Can't resume from a crashed test when " +
+                             "--shuffle is enabled, continue by shuffling " +
+                             "all the tests");
+                DoneTests();
+                return;
+            }
             Shuffle(gURLs);
+        } else if (gStartAfter !== undefined) {
+            // Skip through previously crashed test
+            // We have to do this after chunking so we don't break the numbers
+            var crash_idx = gURLs.map(function(url) {
+                return url['url1']['spec'];
+            }).indexOf(gStartAfter);
+            if (crash_idx == -1) {
+                throw "Can't find the previously crashed test";
+            }
+            gURLs = gURLs.slice(crash_idx + 1);
         }
 
         gTotalTests = gURLs.length;
 
         if (!gTotalTests)
             throw "No tests to run";
 
         gURICanvases = {};
--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -231,22 +231,25 @@ class RemoteReftest(RefTest):
 
         options.xrePath = remoteXrePath
         options.utilityPath = remoteUtilityPath
         return 0
 
     def stopWebServer(self, options):
         self.server.stop()
 
-    def createReftestProfile(self, options, manifest):
+    def createReftestProfile(self, options, manifest, startAfter=None):
         profile = RefTest.createReftestProfile(self,
                                                options,
                                                manifest,
                                                server=options.remoteWebServer,
                                                port=options.httpPort)
+        if startAfter is not None:
+            print ("WARNING: Continuing after a crash is not supported for remote "
+                   "reftest yet.")
         profileDir = profile.profile
 
         prefs = {}
         prefs["app.update.url.android"] = ""
         prefs["browser.firstrun.show.localepicker"] = False
         prefs["reftest.remote"] = True
         prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
 
@@ -313,17 +316,17 @@ class RemoteReftest(RefTest):
                                         binary,
                                         profile.profile,
                                         cmdargs,
                                         utilityPath=options.utilityPath,
                                         xrePath=options.xrePath,
                                         debuggerInfo=debuggerInfo,
                                         symbolsPath=symbolsPath,
                                         timeout=timeout)
-        return status
+        return status, None
 
     def cleanup(self, profileDir):
         # Pull results back from device
         if self.remoteLogFile and \
                 self._devicemanager.fileExists(self.remoteLogFile):
             self._devicemanager.getFile(self.remoteLogFile, self.localLogName)
         else:
             print "WARNING: Unable to retrieve log file (%s) from remote " \
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -242,17 +242,17 @@ class RefTest(object):
             path = os.path.split(path)[0]
         mozinfo.find_and_update_from_json(*dirs)
 
     def getFullPath(self, path):
         "Get an absolute path relative to self.oldcwd."
         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 
     def createReftestProfile(self, options, manifests, server='localhost', port=0,
-                             profile_to_clone=None):
+                             profile_to_clone=None, startAfter=None):
         """Sets up a profile for reftest.
 
         :param options: Object containing command line options
         :param manifests: Dictionary of the form {manifest_path: [filters]}
         :param server: Server name to use for http tests
         :param profile_to_clone: Path to a profile to use as the basis for the
                                  test profile
         """
@@ -281,16 +281,19 @@ class RefTest(object):
         if options.runUntilFailure:
             prefs['reftest.runUntilFailure'] = True
         if options.cleanupCrashes:
             prefs['reftest.cleanupPendingCrashes'] = True
         prefs['reftest.focusFilterMode'] = options.focusFilterMode
         prefs['reftest.logLevel'] = options.log_tbpl_level or 'info'
         prefs['reftest.manifests'] = json.dumps(manifests)
 
+        if startAfter is not None:
+            prefs['reftest.startAfter'] = startAfter
+
         if options.e10s:
             prefs['browser.tabs.remote.autostart'] = True
             prefs['extensions.e10sBlocksEnabling'] = False
 
         # Bug 1262954: For winXP + e10s disable acceleration
         if platform.system() in ("Windows", "Microsoft") and \
            '5.1' in platform.version() and options.e10s:
             prefs['layers.acceleration.disabled'] = True
@@ -344,17 +347,18 @@ class RefTest(object):
         kwargs = {'addons': addons,
                   'preferences': prefs,
                   'locations': locations}
         if profile_to_clone:
             profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
         else:
             profile = mozprofile.Profile(**kwargs)
 
-        options.extraProfileFiles.append(os.path.join(here, 'chrome'))
+        if os.path.join(here, 'chrome') not in options.extraProfileFiles:
+            options.extraProfileFiles.append(os.path.join(here, 'chrome'))
 
         self.copyExtraFilesToProfile(options, profile)
         return profile
 
     def environment(self, **kwargs):
         kwargs['log'] = self.log
         return test_environment(**kwargs)
 
@@ -651,56 +655,75 @@ class RefTest(object):
             self.lastTestSeen = 'Main app process exited normally'
 
         crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'),
                                        symbolsPath, test=self.lastTestSeen)
 
         runner.cleanup()
         if not status and crashed:
             status = 1
-        return status
+        return status, self.lastTestSeen
 
     def runSerialTests(self, manifests, options, cmdargs=None):
         debuggerInfo = None
         if options.debugger:
             debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
                                                       options.debuggerInteractive)
 
         profileDir = None
-        try:
-            if cmdargs is None:
-                cmdargs = []
+        startAfter = None  # When the previous run crashed, we skip the tests we ran before
+        prevStartAfter = None
+        status = 1  # Just to start the loop
+        while status != 0:
+            try:
+                if cmdargs is None:
+                    cmdargs = []
 
-            if self.use_marionette:
-                cmdargs.append('-marionette')
+                if self.use_marionette:
+                    cmdargs.append('-marionette')
 
-            profile = self.createReftestProfile(options, manifests)
-            profileDir = profile.profile  # name makes more sense
+                profile = self.createReftestProfile(options,
+                                                    manifests,
+                                                    startAfter=startAfter)
+                profileDir = profile.profile  # name makes more sense
 
-            # browser environment
-            browserEnv = self.buildBrowserEnv(options, profileDir)
+                # browser environment
+                browserEnv = self.buildBrowserEnv(options, profileDir)
 
-            self.log.info("Running with e10s: {}".format(options.e10s))
-            status = self.runApp(profile,
-                                 binary=options.app,
-                                 cmdargs=cmdargs,
-                                 # give the JS harness 30 seconds to deal with
-                                 # its own timeouts
-                                 env=browserEnv,
-                                 timeout=options.timeout + 30.0,
-                                 symbolsPath=options.symbolsPath,
-                                 options=options,
-                                 debuggerInfo=debuggerInfo)
-            self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
-            mozleak.process_leak_log(self.leakLogFile,
-                                     leak_thresholds=options.leakThresholds,
-                                     stack_fixer=get_stack_fixer_function(options.utilityPath,
-                                                                          options.symbolsPath))
-        finally:
-            self.cleanup(profileDir)
+                self.log.info("Running with e10s: {}".format(options.e10s))
+                status, startAfter = self.runApp(profile,
+                                                 binary=options.app,
+                                                 cmdargs=cmdargs,
+                                                 # give the JS harness 30 seconds to deal with
+                                                 # its own timeouts
+                                                 env=browserEnv,
+                                                 timeout=options.timeout + 30.0,
+                                                 symbolsPath=options.symbolsPath,
+                                                 options=options,
+                                                 debuggerInfo=debuggerInfo)
+                self.log.info("Process mode: {}".format('e10s' if options.e10s else 'non-e10s'))
+                mozleak.process_leak_log(self.leakLogFile,
+                                         leak_thresholds=options.leakThresholds,
+                                         stack_fixer=get_stack_fixer_function(options.utilityPath,
+                                                                              options.symbolsPath))
+                self.cleanup(profileDir)
+                if startAfter is not None and options.shuffle:
+                    self.log.error("Can not resume from a crash with --shuffle "
+                                   "enabled. Please consider disabling --shuffle")
+                    break
+                if startAfter == prevStartAfter:
+                    # If the test stuck on the same test, or there the crashed
+                    # test appeared more then once, stop
+                    self.log.error("Force stop because we keep running into "
+                                   "test \"{}\"".format(startAfter))
+                    break
+                prevStartAfter = startAfter
+                # TODO: we need to emit an SUITE-END log if it crashed
+            finally:
+                self.cleanup(profileDir)
         return status
 
     def copyExtraFilesToProfile(self, options, profile):
         "Copy extra files or dirs specified on the command line to the testing profile."
         profileDir = profile.profile
         if not os.path.exists(os.path.join(profileDir, "hyphenation")):
             os.makedirs(os.path.join(profileDir, "hyphenation"))
         for f in options.extraProfileFiles: