Bug 1368342 - Add mozlog logger that goes via a queue, r=ahal draft
authorJames Graham <james@hoppipolla.co.uk>
Fri, 26 May 2017 12:53:00 +0100
changeset 585737 bc107f4d856846763ca838bc1cba536a84fe5f95
parent 585736 54a6e472d98602427c8c91a70cd72832034cdf5a
child 630788 dea5d330e3fc819865b5ed5f40708f8ed5816157
push id61183
push userbmo:james@hoppipolla.co.uk
push dateSun, 28 May 2017 15:41:41 +0000
reviewersahal
bugs1368342
milestone55.0a1
Bug 1368342 - Add mozlog logger that goes via a queue, r=ahal This allows subprocesses to log to a shared stream via a queue, so that we avoid the overhead of a multiprocessing Lock around all log access, but still avoid races where two processes try to log simultaneously. It's mostly useful where one process is responsible for the majority of logging, but some messages will be generated in child processes. MozReview-Commit-ID: ABl6cvpb6qI
testing/mozbase/mozlog/mozlog/proxy.py
testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
--- a/testing/mozbase/mozlog/mozlog/proxy.py
+++ b/testing/mozbase/mozlog/mozlog/proxy.py
@@ -1,13 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from .structuredlog import get_default_logger
+from multiprocessing import Queue
+from threading import Thread
+
+from .structuredlog import get_default_logger, StructuredLogger
 
 
 class ProxyLogger(object):
     """
     A ProxyLogger behaves like a
     :class:`mozlog.structuredlog.StructuredLogger`.
 
     Each method and attribute access will be forwarded to the underlying
@@ -28,8 +31,48 @@ class ProxyLogger(object):
         return getattr(self.logger, name)
 
 
 def get_proxy_logger(component=None):
     """
     Returns a :class:`ProxyLogger` for the given component.
     """
     return ProxyLogger(component)
+
+
+class QueuedProxyLogger(StructuredLogger):
+    """Logger that logs via a queue.
+
+    This is intended for multiprocessing use cases where there are
+    some subprocesses which want to share a log handler with the main thread,
+    without the overhead of having a multiprocessing lock for all logger
+    access."""
+
+    threads = {}
+
+    def __init__(self, logger):
+        StructuredLogger.__init__(self, logger.name)
+        if logger.name not in self.threads:
+            self.threads[logger.name] = LogQueueThread(Queue(), logger)
+            self.threads[logger.name].start()
+        self.queue = self.threads[logger.name].queue
+
+    def _handle_log(self, data):
+        self.queue.put(data)
+
+
+class LogQueueThread(Thread):
+    def __init__(self, queue, logger):
+        self.queue = queue
+        self.logger = logger
+        Thread.__init__(self, name="Thread-Log")
+        self.daemon = True
+
+    def run(self):
+        while True:
+            try:
+                msg = self.queue.get()
+            except (EOFError, IOError):
+                break
+            if msg is None:
+                break
+            else:
+                self.logger._handle_log(msg)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
@@ -1,17 +1,17 @@
 import json
 import os
 import multiprocessing
 import signal
 import socket
 import sys
 import time
 
-from mozlog import get_default_logger, handlers
+from mozlog import get_default_logger, handlers, proxy
 
 from wptlogging import LogLevelRewriter
 
 here = os.path.split(__file__)[0]
 
 serve = None
 sslutils = None
 
@@ -163,16 +163,18 @@ class TestEnvironment(object):
     def setup_server_logging(self):
         server_logger = get_default_logger(component="wptserve")
         assert server_logger is not None
         log_filter = handlers.LogLevelFilter(lambda x:x, "info")
         # Downgrade errors to warnings for the server
         log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
         server_logger.component_filter = log_filter
 
+        server_logger = proxy.QueuedProxyLogger(server_logger)
+
         try:
             #Set as the default logger for wptserve
             serve.set_logger(server_logger)
             serve.logger = server_logger
         except Exception:
             # This happens if logging has already been set up for wptserve
             pass