Bug 1325148 - Temporarily disable mochitest leakchecking for directories containing known leaks, r?jmaher draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 21 Dec 2016 16:22:50 -0500
changeset 454214 e31cd670215562f386afed4fb340078fa0c996e2
parent 454184 a70de09e4121feeacae4a884aa265ed4a5f4b74a
child 540654 ee80288fd7613260b5844a0c19a419de683882fb
push id39871
push userahalberstadt@mozilla.com
push dateWed, 28 Dec 2016 16:23:38 +0000
reviewersjmaher
bugs1325148
milestone53.0a1
Bug 1325148 - Temporarily disable mochitest leakchecking for directories containing known leaks, r?jmaher Leak checking in mochitest was failing to turn the jobs orange for a period of time. Several leaks slipped through in the interim. In order to land the fix ASAP to avoid further bustage, these leaks are being whitelisted. See the patch or the dependency graph of bug 1325148 for the list of leaks/bugs that are being disabled. While these bugs won't show up in treeherder, they are a priority to fix, as they could represent leaks that are already landed in the code. MozReview-Commit-ID: JY7Gpvswp4r
browser/extensions/shield-recipe-client/test/browser.ini
testing/mochitest/runtests.py
--- a/browser/extensions/shield-recipe-client/test/browser.ini
+++ b/browser/extensions/shield-recipe-client/test/browser.ini
@@ -1,5 +1,6 @@
 [browser_driver_uuids.js]
 [browser_env_expressions.js]
 [browser_EventEmitter.js]
 [browser_Storage.js]
 [browser_Heartbeat.js]
+skip-if = true # bug 1325409
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -819,16 +819,17 @@ class MochitestDesktop(object):
         self.countpass = 0
         self.countfail = 0
         self.counttodo = 0
 
         self.expectedError = {}
         self.result = {}
 
         self.start_script = os.path.join(here, 'start_desktop.js')
+        self.disable_leak_checking = False
 
     def update_mozinfo(self):
         """walk up directories to find mozinfo.json update the info"""
         # TODO: This should go in a more generic place, e.g. mozinfo
 
         path = SCRIPT_DIR
         dirs = set()
         while path != os.path.expanduser('~'):
@@ -1490,17 +1491,18 @@ toolbar#nav-bar {
                 dict(
                     parseKeyValue(
                         options.environment,
                         context='--setenv')))
         except KeyValueParseError as e:
             self.log.error(str(e))
             return None
 
-        browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+        if not self.disable_leak_checking:
+            browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
 
         try:
             gmp_path = self.getGMPPluginPath(options)
             if gmp_path is not None:
                 browserEnv["MOZ_GMP_PATH"] = gmp_path
         except EnvironmentError:
             self.log.error('Could not find path to gmp-fake plugin!')
             return None
@@ -1934,22 +1936,23 @@ toolbar#nav-bar {
             cmd = os.path.abspath(app)
             args = list(extraArgs)
             args.append('-marionette')
             # TODO: mozrunner should use -foreground at least for mac
             # https://bugzilla.mozilla.org/show_bug.cgi?id=916512
             args.append('-foreground')
             self.start_script_args.append(testUrl or 'about:blank')
 
-            if detectShutdownLeaks:
+            if detectShutdownLeaks and not self.disable_leak_checking:
                 shutdownLeaks = ShutdownLeaks(self.log)
             else:
                 shutdownLeaks = None
 
-            if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac):
+            if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac) \
+                    and not self.disable_leak_checking:
                 lsanLeaks = LSANLeaks(self.log)
             else:
                 lsanLeaks = None
 
             # create an instance to process the output
             outputHandler = self.OutputHandler(
                 harness=self,
                 utilityPath=utilityPath,
@@ -2196,16 +2199,46 @@ toolbar#nav-bar {
             return self.runMochitests(options, testsToRun)
 
         # code for --run-by-dir
         dirs = self.getDirectories(options)
 
         result = 1  # default value, if no tests are run.
         for d in dirs:
             print "dir: %s" % d
+
+            # BEGIN LEAKCHECK HACK
+            # Leak checking was broken in mochitest unnoticed for a length of time. During
+            # this time, several leaks slipped through. The leak checking was fixed by bug
+            # 1325148, but it couldn't land until all the regressions were also fixed or
+            # backed out. Rather than waiting and risking new regressions, in the meantime
+            # this code will selectively disable leak checking on flavors/directories where
+            # known regressions exist. At least this way we can prevent further damage while
+            # they get fixed.
+
+            info = mozinfo.info
+            skip_leak_conditions = [
+                (options.flavor in ('browser', 'chrome', 'plain') and d.startswith('toolkit/components/extensions/test/mochitest'), 'bug 1325158'),  # noqa
+                (info['debug'] and options.flavor == 'browser' and d.startswith('browser/components/extensions/test/browser'), 'bug 1325141'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'dom/animation/test/css-animations', 'bug 1325277'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'dom/tests/mochitest/gamepad' and info['os'] == 'win', 'bug 1324592'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'toolkit/components/prompts/test' and info['os'] == 'mac', 'bug 1325275'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'tests/dom/xhr/tests', 'bug 1325438'),  # noqa
+            ]
+
+            for condition, reason in skip_leak_conditions:
+                if condition:
+                    self.log.warning('WARNING | disabling leakcheck due to {}'.format(reason))
+                    self.disable_leak_checking = True
+                    break
+            else:
+                self.disable_leak_checking = False
+
+            # END LEAKCHECK HACK
+
             tests_in_dir = [t for t in testsToRun if os.path.dirname(t) == d]
 
             # If we are using --run-by-dir, we should not use the profile path (if) provided
             # by the user, since we need to create a new directory for each run. We would face
             # problems if we use the directory provided by the user.
             result = self.runMochitests(options, tests_in_dir)
 
             # Dump the logging buffer