Bug 1313265 - Ensure unstructured logs containing TEST-UNEXPECTED-FAIL fail the job desktop_unittest.py, r?jgraham draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 27 Oct 2016 15:11:26 -0400
changeset 430437 b4e073da7ad16ffc9e657160105b1d5fdd669bda
parent 430433 6788a1ac65f71f9d9ea5553b496539fe7a1fa83e
child 535209 03cfc322683eea10fb1562f2111f52def49a215f
push id33831
push userahalberstadt@mozilla.com
push dateThu, 27 Oct 2016 20:19:01 +0000
reviewersjgraham
bugs1313265
milestone52.0a1
Bug 1313265 - Ensure unstructured logs containing TEST-UNEXPECTED-FAIL fail the job desktop_unittest.py, r?jgraham I haven't seen a case of this happening in the wild, but I believe it is possible for a test to dump "TEST-UNEXPECTED-FAIL" directly to the log and mozharness using the StructuredOutputParser would not pick it up. This patch is just me being extra careful to flag potential errors like that. I'm not sure they even exist. This patch also purposefully uses substr to avoid requiring said string to show up at the beginning, which should avoid certain prefix issues we've run into in the past. MozReview-Commit-ID: 99n9YizlEDH
testing/mozharness/mozharness/mozilla/testing/errors.py
testing/mozharness/scripts/desktop_unittest.py
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ b/testing/mozharness/mozharness/mozilla/testing/errors.py
@@ -101,14 +101,19 @@ TinderBoxPrintRe = {
         'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost)''')
     },
 }
 
 TestPassed = [
     {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
 ]
 
+HarnessErrorList = [
+    {'substr': 'TEST-UNEXPECTED', 'level': ERROR, },
+    {'substr': 'PROCESS-CRASH', 'level': ERROR, },
+]
+
 LogcatErrorList = [
     {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
     {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
     {'substr': '[JavaScript Error:', 'level': WARNING},
     {'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'},
 ]
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -24,16 +24,17 @@ sys.path.insert(1, os.path.dirname(sys.p
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.log import INFO, ERROR
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.buildbot import TBPL_EXCEPTION
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 
 SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
@@ -671,20 +672,17 @@ class DesktopUnittest(TestingMixin, Merc
                                               try_options,
                                               str_format_values=replace_dict))
                 cmd.extend(self.query_tests_args(tests_list,
                                                  try_tests,
                                                  str_format_values=replace_dict))
 
                 suite_name = suite_category + '-' + suite
                 tbpl_status, log_level = None, None
-                error_list = BaseErrorList + [{
-                    'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
-                    'level': ERROR,
-                }]
+                error_list = BaseErrorList + HarnessErrorList
                 parser = self.get_test_output_parser(suite_category,
                                                      flavor=flavor,
                                                      config=self.config,
                                                      error_list=error_list,
                                                      log_obj=self.log_obj)
 
                 if suite_category == "reftest":
                     ref_formatter = imp.load_source(