Bug 1375104 - Combine standard and Gecko log for Marionette. draft
authorHenrik Skupin <mail@hskupin.info>
Sat, 24 Jun 2017 12:00:56 +0200
changeset 601418 d6cd4f572dcc822ab6acbbad42fb1d89f98fd9d6
parent 601255 cc903e3f61894e60c3b0efaf05e9a446d1d85888
child 635243 9573a8de03a2cd7c4208b7945d2545d88ef2679c
push id66044
push userbmo:hskupin@gmail.com
push dateWed, 28 Jun 2017 18:46:26 +0000
bugs1375104
milestone56.0a1
Bug 1375104 - Combine standard and Gecko log for Marionette. In automation we should combine the standard logging and the gecko log by default to ease the investigation of test failures. It will also provide crash/assertion output without having to search for it in other log files. Also the custom error list has been replaced with the default base and harness error lists as used by other harnesses, which prevents false assumptions by the parser when the trace log contains returns of expected errors. MozReview-Commit-ID: 1rQ6maOqD3V
testing/mozharness/scripts/marionette.py
--- a/testing/mozharness/scripts/marionette.py
+++ b/testing/mozharness/scripts/marionette.py
@@ -8,29 +8,31 @@
 import copy
 import os
 import re
 import sys
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
-from mozharness.base.errors import TarErrorList
+from mozharness.base.errors import BaseErrorList, TarErrorList
 from mozharness.base.log import INFO, ERROR, WARNING
 from mozharness.base.script import PreScriptAction
 from mozharness.base.transfer import TransferMixin
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.testing.errors import LogcatErrorList
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 
 # TODO: we could remove emulator specific code after B2G ICS emulator buildbot
 #       builds is turned off, Bug 1209180.
 
 
 class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMixin, CodeCoverageMixin):
     config_options = [[
@@ -110,25 +112,16 @@ class MarionetteTest(TestingMixin, Mercu
         "dest": "enable_webrender",
         "default": False,
         "help": "Tries to enable the WebRender compositor."
         }
      ]] + copy.deepcopy(testing_config_options) \
         + copy.deepcopy(blobupload_config_options) \
         + copy.deepcopy(code_coverage_config_options)
 
-    error_list = [
-        {'substr': 'FAILED (errors=', 'level': WARNING},
-        {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
-        {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
-        {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
-        {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''), 'level': ERROR},
-        {'regex': re.compile(r'''(\b((?!Marionette|TestMarionette|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidResponse|Javascript|Timeout|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)\w*)Exception)'''), 'level': ERROR},
-    ]
-
     repos = []
 
     def __init__(self, require_config_file=False):
         super(MarionetteTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'read-buildbot-config',
                          'pull',
@@ -290,18 +283,17 @@ class MarionetteTest(TestingMixin, Mercu
             config_fmt_args['app_arg'] = self.config['app_arg']
 
         if not self.config['e10s']:
             cmd.append('--disable-e10s')
 
         if self.config['headless']:
             cmd.append('--headless')
 
-        cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
-                                                   'gecko.log'))
+        cmd.append('--gecko-log=-')
 
         if self.config.get("structured_output"):
             cmd.append("--log-raw=-")
 
         options_group = self._get_options_group(self.config.get('emulator'))
 
         if options_group not in self.config["suite_definitions"]:
             self.fatal("%s is not defined in the config!" % options_group)
@@ -332,21 +324,22 @@ class MarionetteTest(TestingMixin, Mercu
             env['MOZ_WEBRENDER'] = '1'
 
         if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
             self.mkdir_p(env['MOZ_UPLOAD_DIR'])
         env = self.query_env(partial_env=env)
 
         marionette_parser = self.parser_class(config=self.config,
                                               log_obj=self.log_obj,
-                                              error_list=self.error_list,
+                                              error_list=BaseErrorList + HarnessErrorList,
                                               strict=False)
-        return_code = self.run_command(cmd, env=env,
-                                output_timeout=1000,
-                                output_parser=marionette_parser)
+        return_code = self.run_command(cmd,
+                                       output_timeout=1000,
+                                       output_parser=marionette_parser,
+                                       env=env)
         level = INFO
         tbpl_status, log_level = marionette_parser.evaluate_parser(
             return_code=return_code)
         marionette_parser.append_tinderboxprint_line("marionette")
 
         qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
         if os.path.isfile(qemu):
             self.copyfile(qemu, os.path.join(dirs['abs_blob_upload_dir'],