Bug 1338534 - [mozlog] Improve handling of expected failures in pytest-mozlog. r?ahal draft
authorDave Hunt <dhunt@mozilla.com>
Tue, 14 Feb 2017 12:07:36 +0000
changeset 485056 6c5f52c5c839f50597cffc7dce60f02cbbb8db5e
parent 485055 88f55c5a10d4f671e258dcfbf13ecc9f494cf832
child 485057 1960789da0cf58aa7df5da0a0a0698a38e43da77
push id45612
push userdhunt@mozilla.com
push dateThu, 16 Feb 2017 01:41:44 +0000
reviewersahal
bugs1338534
milestone54.0a1
Bug 1338534 - [mozlog] Improve handling of expected failures in pytest-mozlog. r?ahal Include failure messages and stack traces even when failures are expected. MozReview-Commit-ID: HSNYfbTEH9O
testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
--- a/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
+++ b/testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py
@@ -68,36 +68,33 @@ class MozLog(object):
         self.logger.test_start(test=nodeid)
 
     def pytest_runtest_logreport(self, report):
         '''Called 3 times per test (setup, call, teardown), indicated by report.when'''
         test = report.nodeid
         status = expected = 'PASS'
         message = stack = None
         if hasattr(report, 'wasxfail'):
-            # Pytest reporting for xfail tests is somewhat counterinutitive:
-            # If an xfail test fails as expected, its 'call' report has .skipped,
-            # so we record status FAIL (== expected) and log an expected result.
-            # If an xfail unexpectedly passes, the 'call' report has .failed (Pytest 2)
-            # or .passed (Pytest 3), so we leave status as PASS (!= expected)
-            # to log an unexpected result.
             expected = 'FAIL'
-            if report.skipped:  # indicates expected failure (passing test)
-                status = 'FAIL'
-        elif report.failed:
+        if report.failed:
             status = 'FAIL' if report.when == 'call' else 'ERROR'
-            try:
-                crash = report.longrepr.reprcrash  # here longrepr is a ReprExceptionInfo
-                message = "{0} (line {1})".format(crash.message, crash.lineno)
-                stack = report.longrepr.reprtraceback
-            except AttributeError:
+        if report.skipped:
+            status = 'SKIP' if not hasattr(report, 'wasxfail') else 'FAIL'
+        if report.longrepr is not None:
+            if isinstance(report.longrepr, basestring):
                 # When using pytest-xdist, longrepr is serialised as a str
                 message = stack = report.longrepr
-        elif report.skipped:  # indicates true skip
-            status = expected = 'SKIP'
-            message = report.longrepr[-1]  # here longrepr is a tuple (file, lineno, reason)
+            else:
+                try:
+                    # For failures, longrepr is a ReprExceptionInfo
+                    crash = report.longrepr.reprcrash
+                    message = "{0} (line {1})".format(crash.message, crash.lineno)
+                    stack = report.longrepr.reprtraceback
+                except AttributeError:
+                    # For skips, longrepr is a tuple of (file, lineno, reason)
+                    message = report.longrepr[-1]
         if status != expected or expected != 'PASS':
             self.results[test] = (status, expected, message, stack)
         if report.when == 'teardown':
             defaults = ('PASS', 'PASS', None, None)
             status, expected, message, stack = self.results.get(test, defaults)
             self.logger.test_end(test=test, status=status, expected=expected,
                                  message=message, stack=stack)