Bug 1368342 - Add mozlog logger that goes via a queue, r=ahal
This allows subprocesses to log to a shared stream via a queue, so that we
avoid the overhead of a multiprocessing Lock around all log access, but still
avoid races where two processes try to log simultaneously. It's mostly useful
where one process is responsible for the majority of logging, but some messages
will be generated in child processes.
MozReview-Commit-ID: ABl6cvpb6qI
--- a/testing/mozbase/mozlog/mozlog/proxy.py
+++ b/testing/mozbase/mozlog/mozlog/proxy.py
@@ -1,13 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-from .structuredlog import get_default_logger
+from multiprocessing import Queue
+from threading import Thread
+
+from .structuredlog import get_default_logger, StructuredLogger
class ProxyLogger(object):
"""
A ProxyLogger behaves like a
:class:`mozlog.structuredlog.StructuredLogger`.
Each method and attribute access will be forwarded to the underlying
@@ -28,8 +31,48 @@ class ProxyLogger(object):
return getattr(self.logger, name)
def get_proxy_logger(component=None):
"""
Returns a :class:`ProxyLogger` for the given component.
"""
return ProxyLogger(component)
+
+
+class QueuedProxyLogger(StructuredLogger):
+ """Logger that logs via a queue.
+
+ This is intended for multiprocessing use cases where there are
+ some subprocesses which want to share a log handler with the main thread,
+ without the overhead of having a multiprocessing lock for all logger
+ access."""
+
+ threads = {}
+
+ def __init__(self, logger):
+ StructuredLogger.__init__(self, logger.name)
+ if logger.name not in self.threads:
+ self.threads[logger.name] = LogQueueThread(Queue(), logger)
+ self.threads[logger.name].start()
+ self.queue = self.threads[logger.name].queue
+
+ def _handle_log(self, data):
+ self.queue.put(data)
+
+
+class LogQueueThread(Thread):
+ def __init__(self, queue, logger):
+ self.queue = queue
+ self.logger = logger
+ Thread.__init__(self, name="Thread-Log")
+ self.daemon = True
+
+ def run(self):
+ while True:
+ try:
+ msg = self.queue.get()
+ except (EOFError, IOError):
+ break
+ if msg is None:
+ break
+ else:
+ self.logger._handle_log(msg)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
@@ -1,17 +1,17 @@
import json
import os
import multiprocessing
import signal
import socket
import sys
import time
-from mozlog import get_default_logger, handlers
+from mozlog import get_default_logger, handlers, proxy
from wptlogging import LogLevelRewriter
here = os.path.split(__file__)[0]
serve = None
sslutils = None
@@ -163,16 +163,18 @@ class TestEnvironment(object):
def setup_server_logging(self):
server_logger = get_default_logger(component="wptserve")
assert server_logger is not None
log_filter = handlers.LogLevelFilter(lambda x:x, "info")
# Downgrade errors to warnings for the server
log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
server_logger.component_filter = log_filter
+ server_logger = proxy.QueuedProxyLogger(server_logger)
+
try:
#Set as the default logger for wptserve
serve.set_logger(server_logger)
serve.logger = server_logger
except Exception:
# This happens if logging has already been set up for wptserve
pass