Bug 1415614 - Add an API to log all structured messages; r?build draft
authorGregory Szorc <gps@mozilla.com>
Thu, 09 Nov 2017 15:09:52 -0800
changeset 695860 f50dbbe024e50dfc2d3ea4a46c328ef753e6bab6
parent 695703 dca383e5f5b770fa3adb09d753897633258302a1
child 695861 ac1ce7a17b7be9027b05901c7d4fa79c0cfa3674
push id88570
push userbmo:gps@mozilla.com
push dateThu, 09 Nov 2017 23:11:44 +0000
reviewersbuild
bugs1415614
milestone58.0a1
Bug 1415614 - Add an API to log all structured messages; r?build Currently, marking a logger as a structured logger will require a subsequent function call in order for the logger to be hooked up to active handlers. This behavior is not intuitive and makes it easy to not have handlers for newly-registered loggers. This means messages may not be logged anywhere. In addition, we have to manually specify which named loggers to enable structured logging for. This can be annoying. We change the behavior of register_structured_logger() to automatically add existing terminal and json handlers to the logger being marked as structured. We also introduce an API to enable structured logging for all loggers. Existing consumers of registered_structured_logger() in mozbuild have been updated to use this API. A new consumer has been added for the `mach configure` command because it should have been there before. We stop short of making enable_all_structured_loggers() the default. This is because various commands interact with the log manager in ways that will result in duplicate logging of messages and dropping of structured messages. There is a bit of a rabbit hole here and addressing it can be done as a follow-up. MozReview-Commit-ID: 1aU6eJvTSMP
python/mach/mach/logging.py
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mach/mach/logging.py
+++ b/python/mach/mach/logging.py
@@ -251,15 +251,48 @@ class LoggingManager(object):
             self.root_logger.addHandler(self.terminal_handler)
 
     def disable_unstructured(self):
         """Disable logging of unstructured messages."""
         if self.terminal_handler:
             self.terminal_handler.removeFilter(self.structured_filter)
             self.root_logger.removeHandler(self.terminal_handler)
 
-    def register_structured_logger(self, logger):
+    def register_structured_logger(self, logger, terminal=True, json=True):
         """Register a structured logger.
 
         This needs to be called for all structured loggers that don't chain up
         to the mach logger in order for their output to be captured.
         """
         self.structured_loggers.append(logger)
+
+        if terminal and self.terminal_handler:
+            logger.addHandler(self.terminal_handler)
+
+        if json:
+            for handler in self.json_handlers:
+                logger.addHandler(handler)
+
+    def enable_all_structured_loggers(self, terminal=True, json=True):
+        """Enable logging of all structured messages from all loggers.
+
+        ``terminal`` and ``json`` determine which log handlers to operate
+        on. By default, all known handlers are operated on.
+        """
+        # Remove current handlers from all loggers so we don't double
+        # register handlers.
+        for logger in self.root_logger.manager.loggerDict.values():
+            # Some entries might be logging.PlaceHolder.
+            if not isinstance(logger, logging.Logger):
+                continue
+
+            if terminal:
+                logger.removeHandler(self.terminal_handler)
+
+            if json:
+                for handler in self.json_handlers:
+                    logger.removeHandler(handler)
+
+        # Wipe out existing registered structured loggers since they
+        # all propagate to root logger.
+        self.structured_loggers = []
+        self.register_structured_logger(self.root_logger, terminal=terminal,
+                                        json=json)
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -389,17 +389,17 @@ class Build(MachCommandBase):
         """
         import which
         from mozbuild.controller.building import BuildMonitor
         from mozbuild.util import (
             mkdir,
             resolve_target_to_make,
         )
 
-        self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
+        self.log_manager.enable_all_structured_loggers()
 
         warnings_path = self._get_state_filename('warnings.json')
         monitor = self._spawn(BuildMonitor)
         monitor.init(warnings_path)
         ccache_start = monitor.ccache_stats()
         footer = BuildProgressFooter(self.log_manager.terminal, monitor)
 
         # Disable indexing in objdir because it is not necessary and can slow
@@ -685,16 +685,18 @@ class Build(MachCommandBase):
 
         return status
 
     @Command('configure', category='build',
         description='Configure the tree (run configure and config.status).')
     @CommandArgument('options', default=None, nargs=argparse.REMAINDER,
                      help='Configure options')
     def configure(self, options=None, buildstatus_messages=False, line_handler=None):
+        self.log_manager.enable_all_structured_loggers()
+
         def on_line(line):
             self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
 
         line_handler = line_handler or on_line
 
         options = ' '.join(shell_quote(o) for o in options or ())
         append_env = {b'CONFIGURE_ARGS': options.encode('utf-8')}
 
@@ -2209,16 +2211,18 @@ class StaticAnalysis(MachCommandBase):
                      help='Try to autofix errors detected by clang-tidy checkers.')
     @CommandArgument('--header-filter', '-h-f', default='', metavar='header_filter',
                      help='Regular expression matching the names of the headers to '
                           'output diagnostics from. Diagnostics from the main file '
                           'of each translation unit are always displayed')
     def check(self, source=None, jobs=2, strip=1, verbose=False,
               checks='-*', fix=False, header_filter=''):
         self._set_log_level(verbose)
+        self.log_manager.enable_all_structured_loggers()
+
         rc = self._build_compile_db(verbose=verbose)
         if rc != 0:
             return rc
 
         rc = self._build_export(jobs=jobs, verbose=verbose)
         if rc != 0:
             return rc
 
@@ -2241,18 +2245,16 @@ class StaticAnalysis(MachCommandBase):
         # When no value is specified the default value is considered to be the source
         # in order to limit the dianostic message to the source files or folders.
         common_args.append('-header-filter=%s' %
                            (header_filter if len(header_filter) else ''.join(source)))
 
         if fix:
             common_args.append('-fix')
 
-        self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
-
         compile_db = json.loads(open(self._compile_db, 'r').read())
         total = 0
         import re
         name_re = re.compile('(' + ')|('.join(source) + ')')
         for f in compile_db:
             if name_re.search(f['file']):
                 total = total + 1