Bug 1313265 - Ensure unstructured logs containing TEST-UNEXPECTED-FAIL fail the job desktop_unittest.py, r?jgraham
I haven't seen a case of this happening in the wild, but I believe it is possible for a test to dump "TEST-UNEXPECTED-FAIL"
directly to the log and mozharness using the StructuredOutputParser would not pick it up. This patch is just me being extra
careful to flag potential errors like that. I'm not sure they even exist.
This patch also purposefully uses substr to avoid requiring said string to show up at the beginning, which should avoid
certain prefix issues we've run into in the past.
MozReview-Commit-ID: 99n9YizlEDH
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ b/testing/mozharness/mozharness/mozilla/testing/errors.py
@@ -101,14 +101,19 @@ TinderBoxPrintRe = {
'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost)''')
},
}
TestPassed = [
{'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
]
+HarnessErrorList = [
+ {'substr': 'TEST-UNEXPECTED', 'level': ERROR, },
+ {'substr': 'PROCESS-CRASH', 'level': ERROR, },
+]
+
LogcatErrorList = [
{'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
{'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
{'substr': '[JavaScript Error:', 'level': WARNING},
{'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'},
]
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -24,16 +24,17 @@ sys.path.insert(1, os.path.dirname(sys.p
from mozharness.base.errors import BaseErrorList
from mozharness.base.log import INFO, ERROR
from mozharness.base.script import PreScriptAction
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.buildbot import TBPL_EXCEPTION
from mozharness.mozilla.mozbase import MozbaseMixin
from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
from mozharness.mozilla.testing.codecoverage import (
CodeCoverageMixin,
code_coverage_config_options
)
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
@@ -671,20 +672,17 @@ class DesktopUnittest(TestingMixin, Merc
try_options,
str_format_values=replace_dict))
cmd.extend(self.query_tests_args(tests_list,
try_tests,
str_format_values=replace_dict))
suite_name = suite_category + '-' + suite
tbpl_status, log_level = None, None
- error_list = BaseErrorList + [{
- 'regex': re.compile(r'''PROCESS-CRASH.*application crashed'''),
- 'level': ERROR,
- }]
+ error_list = BaseErrorList + HarnessErrorList
parser = self.get_test_output_parser(suite_category,
flavor=flavor,
config=self.config,
error_list=error_list,
log_obj=self.log_obj)
if suite_category == "reftest":
ref_formatter = imp.load_source(