--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -3,32 +3,32 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
import getpass
import json
import logging
import os
-import platform
import subprocess
import sys
import time
import which
from collections import (
namedtuple,
OrderedDict,
)
try:
import psutil
except Exception:
psutil = None
+from mach.mixin.logging import LoggingMixin
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
import mozpack.path as mozpath
from ..base import MozbuildObject
from ..testing import install_test_files
@@ -480,16 +480,285 @@ class BuildMonitor(MozbuildObject):
except which.WhichError:
pass
except ValueError as e:
self.log(logging.WARNING, 'ccache', {'msg': str(e)}, '{msg}')
return ccache_stats
+class TerminalLoggingHandler(logging.Handler):
+ """Custom logging handler that works with terminal window dressing.
+
+ This class should probably live elsewhere, like the mach core. Consider
+ this a proving ground for its usefulness.
+ """
+ def __init__(self):
+ logging.Handler.__init__(self)
+
+ self.fh = sys.stdout
+ self.footer = None
+
+ def flush(self):
+ self.acquire()
+
+ try:
+ self.fh.flush()
+ finally:
+ self.release()
+
+ def emit(self, record):
+ msg = self.format(record)
+
+ self.acquire()
+
+ try:
+ if self.footer:
+ self.footer.clear()
+
+ self.fh.write(msg)
+ self.fh.write('\n')
+
+ if self.footer:
+ self.footer.draw()
+
+ # If we don't flush, the footer may not get drawn.
+ self.fh.flush()
+ finally:
+ self.release()
+
+
+class Footer(object):
+ """Handles display of a footer in a terminal.
+
+ This class implements the functionality common to all mach commands
+ that render a footer.
+ """
+
+ def __init__(self, terminal):
+ # terminal is a blessings.Terminal.
+ self._t = terminal
+ self._fh = sys.stdout
+
+ def clear(self):
+ """Removes the footer from the current terminal."""
+ self._fh.write(self._t.move_x(0))
+ self._fh.write(self._t.clear_eol())
+
+ def write(self, parts):
+ """Write some output in the footer, accounting for terminal width.
+
+ parts is a list of 2-tuples of (encoding_function, input).
+ None means no encoding."""
+
+ # We don't want to write more characters than the current width of the
+ # terminal otherwise wrapping may result in weird behavior. We can't
+ # simply truncate the line at terminal width characters because a)
+ # non-viewable escape characters count towards the limit and b) we
+ # don't want to truncate in the middle of an escape sequence because
+ # subsequent output would inherit the escape sequence.
+ max_width = self._t.width
+ written = 0
+ write_pieces = []
+ for part in parts:
+ try:
+ func, part = part
+ encoded = getattr(self._t, func)(part)
+ except ValueError:
+ encoded = part
+
+ len_part = len(part)
+ len_spaces = len(write_pieces)
+ if written + len_part + len_spaces > max_width:
+ write_pieces.append(part[0:max_width - written - len_spaces])
+ written += len_part
+ break
+
+ write_pieces.append(encoded)
+ written += len_part
+
+ with self._t.location():
+ self._t.move(self._t.height-1,0)
+ self._fh.write(' '.join(write_pieces))
+
+
+class BuildProgressFooter(Footer):
+ """Handles display of a build progress indicator in a terminal.
+
+ When mach builds inside a blessings-supported terminal, it will render
+ progress information collected from a BuildMonitor. This class converts the
+ state of BuildMonitor into terminal output.
+ """
+
+ def __init__(self, terminal, monitor):
+ Footer.__init__(self, terminal)
+ self.tiers = monitor.tiers.tier_status.viewitems()
+
+ def draw(self):
+ """Draws this footer in the terminal."""
+
+ if not self.tiers:
+ return
+
+ # The drawn terminal looks something like:
+ # TIER: static export libs tools
+
+ parts = [('bold', 'TIER:')]
+ append = parts.append
+ for tier, status in self.tiers:
+ if status is None:
+ append(tier)
+ elif status == 'finished':
+ append(('green', tier))
+ else:
+ append(('underline_yellow', tier))
+
+ self.write(parts)
+
+
+
+class OutputManager(LoggingMixin):
+ """Handles writing job output to a terminal or log."""
+
+ def __init__(self, log_manager, footer):
+ self.populate_logger()
+
+ self.footer = None
+ terminal = log_manager.terminal
+
+ # TODO convert terminal footer to config file setting.
+ if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None):
+ return
+ if os.environ.get('INSIDE_EMACS', None):
+ return
+
+ self.t = terminal
+ self.footer = footer
+
+ self._handler = TerminalLoggingHandler()
+ self._handler.setFormatter(log_manager.terminal_formatter)
+ self._handler.footer = self.footer
+
+ old = log_manager.replace_terminal_handler(self._handler)
+ self._handler.level = old.level
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.footer:
+ self.footer.clear()
+ # Prevents the footer from being redrawn if logging occurs.
+ self._handler.footer = None
+
+ def write_line(self, line):
+ if self.footer:
+ self.footer.clear()
+
+ print(line)
+
+ if self.footer:
+ self.footer.draw()
+
+ def refresh(self):
+ if not self.footer:
+ return
+
+ self.footer.clear()
+ self.footer.draw()
+
+
+class BuildOutputManager(OutputManager):
+ """Handles writing build output to a terminal, to logs, etc."""
+
+ def __init__(self, log_manager, monitor, footer):
+ self.monitor = monitor
+ OutputManager.__init__(self, log_manager, footer)
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ OutputManager.__exit__(self, exc_type, exc_value, traceback)
+
+ # Ensure the resource monitor is stopped because leaving it running
+ # could result in the process hanging on exit because the resource
+ # collection child process hasn't been told to stop.
+ self.monitor.stop_resource_recording()
+
+
+ def on_line(self, line):
+ warning, state_changed, relevant = self.monitor.on_line(line)
+
+ if relevant:
+ self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+ elif state_changed:
+ have_handler = hasattr(self, 'handler')
+ if have_handler:
+ self.handler.acquire()
+ try:
+ self.refresh()
+ finally:
+ if have_handler:
+ self.handler.release()
+
+
+class StaticAnalysisFooter(Footer):
+ """Handles display of a static analysis progress indicator in a terminal.
+ """
+
+ def __init__(self, terminal, monitor):
+ Footer.__init__(self, terminal)
+ self.monitor = monitor
+
+ def draw(self):
+ """Draws this footer in the terminal."""
+
+ monitor = self.monitor
+ total = monitor.num_files
+ processed = monitor.num_files_processed
+ percent = '(%.2f%%)' % (processed * 100.0 / total)
+ parts = [
+ ('dim', 'Processing'),
+ ('yellow', str(processed)),
+ ('dim', 'of'),
+ ('yellow', str(total)),
+ ('dim', 'files'),
+ ('green', percent)
+ ]
+ if monitor.current_file:
+ parts.append(('bold', monitor.current_file))
+
+ self.write(parts)
+
+
+class StaticAnalysisOutputManager(OutputManager):
+ """Handles writing static analysis output to a terminal."""
+
+ def __init__(self, log_manager, monitor, footer):
+ self.monitor = monitor
+ OutputManager.__init__(self, log_manager, footer)
+
+ def on_line(self, line):
+ warning, relevant = self.monitor.on_line(line)
+
+ if warning:
+ self.log(logging.INFO, 'compiler_warning', warning,
+ 'Warning: {flag} in {filename}: {message}')
+
+ if relevant:
+ self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+ else:
+ have_handler = hasattr(self, 'handler')
+ if have_handler:
+ self.handler.acquire()
+ try:
+ self.refresh()
+ finally:
+ if have_handler:
+ self.handler.release()
+
+
class CCacheStats(object):
"""Holds statistics from ccache.
Instances can be subtracted from each other to obtain differences.
print() or str() the object to show a ``ccache -s`` like output
of the captured stats.
"""
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1,17 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import argparse
import collections
-import errno
import hashlib
import itertools
import json
import logging
import operator
import os
import subprocess
import sys
@@ -24,28 +23,23 @@ from mach.decorators import (
CommandArgument,
CommandArgumentGroup,
CommandProvider,
Command,
SettingsProvider,
SubCommand,
)
-from mach.mixin.logging import LoggingMixin
-
from mach.main import Mach
from mozbuild.base import (
BuildEnvironmentNotFoundException,
MachCommandBase,
MachCommandConditions as conditions,
MozbuildObject,
- MozconfigFindException,
- MozconfigLoadException,
- ObjdirMismatchException,
)
from mozbuild.util import ensureParentDir
from mozbuild.backend import (
backends,
get_backend_class,
)
from mozbuild.shellutil import quote as shell_quote
@@ -86,227 +80,16 @@ If you feel this message is not appropri
please file a Core :: Build Config bug at
https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config
and tell us about your machine and build configuration so we can adjust the
warning heuristic.
===================
'''
-class TerminalLoggingHandler(logging.Handler):
- """Custom logging handler that works with terminal window dressing.
-
- This class should probably live elsewhere, like the mach core. Consider
- this a proving ground for its usefulness.
- """
- def __init__(self):
- logging.Handler.__init__(self)
-
- self.fh = sys.stdout
- self.footer = None
-
- def flush(self):
- self.acquire()
-
- try:
- self.fh.flush()
- finally:
- self.release()
-
- def emit(self, record):
- msg = self.format(record)
-
- self.acquire()
-
- try:
- if self.footer:
- self.footer.clear()
-
- self.fh.write(msg)
- self.fh.write('\n')
-
- if self.footer:
- self.footer.draw()
-
- # If we don't flush, the footer may not get drawn.
- self.fh.flush()
- finally:
- self.release()
-
-
-class Footer(object):
- """Handles display of a footer in a terminal.
-
- This class implements the functionality common to all mach commands
- that render a footer.
- """
-
- def __init__(self, terminal):
- # terminal is a blessings.Terminal.
- self._t = terminal
- self._fh = sys.stdout
-
- def clear(self):
- """Removes the footer from the current terminal."""
- self._fh.write(self._t.move_x(0))
- self._fh.write(self._t.clear_eol())
-
- def write(self, parts):
- """Write some output in the footer, accounting for terminal width.
-
- parts is a list of 2-tuples of (encoding_function, input).
- None means no encoding."""
-
- # We don't want to write more characters than the current width of the
- # terminal otherwise wrapping may result in weird behavior. We can't
- # simply truncate the line at terminal width characters because a)
- # non-viewable escape characters count towards the limit and b) we
- # don't want to truncate in the middle of an escape sequence because
- # subsequent output would inherit the escape sequence.
- max_width = self._t.width
- written = 0
- write_pieces = []
- for part in parts:
- try:
- func, part = part
- encoded = getattr(self._t, func)(part)
- except ValueError:
- encoded = part
-
- len_part = len(part)
- len_spaces = len(write_pieces)
- if written + len_part + len_spaces > max_width:
- write_pieces.append(part[0:max_width - written - len_spaces])
- written += len_part
- break
-
- write_pieces.append(encoded)
- written += len_part
-
- with self._t.location():
- self._t.move(self._t.height-1,0)
- self._fh.write(' '.join(write_pieces))
-
-
-class BuildProgressFooter(Footer):
- """Handles display of a build progress indicator in a terminal.
-
- When mach builds inside a blessings-supported terminal, it will render
- progress information collected from a BuildMonitor. This class converts the
- state of BuildMonitor into terminal output.
- """
-
- def __init__(self, terminal, monitor):
- Footer.__init__(self, terminal)
- self.tiers = monitor.tiers.tier_status.viewitems()
-
- def draw(self):
- """Draws this footer in the terminal."""
-
- if not self.tiers:
- return
-
- # The drawn terminal looks something like:
- # TIER: static export libs tools
-
- parts = [('bold', 'TIER:')]
- append = parts.append
- for tier, status in self.tiers:
- if status is None:
- append(tier)
- elif status == 'finished':
- append(('green', tier))
- else:
- append(('underline_yellow', tier))
-
- self.write(parts)
-
-
-class OutputManager(LoggingMixin):
- """Handles writing job output to a terminal or log."""
-
- def __init__(self, log_manager, footer):
- self.populate_logger()
-
- self.footer = None
- terminal = log_manager.terminal
-
- # TODO convert terminal footer to config file setting.
- if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None):
- return
- if os.environ.get('INSIDE_EMACS', None):
- return
-
- self.t = terminal
- self.footer = footer
-
- self._handler = TerminalLoggingHandler()
- self._handler.setFormatter(log_manager.terminal_formatter)
- self._handler.footer = self.footer
-
- old = log_manager.replace_terminal_handler(self._handler)
- self._handler.level = old.level
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.footer:
- self.footer.clear()
- # Prevents the footer from being redrawn if logging occurs.
- self._handler.footer = None
-
- def write_line(self, line):
- if self.footer:
- self.footer.clear()
-
- print(line)
-
- if self.footer:
- self.footer.draw()
-
- def refresh(self):
- if not self.footer:
- return
-
- self.footer.clear()
- self.footer.draw()
-
-class BuildOutputManager(OutputManager):
- """Handles writing build output to a terminal, to logs, etc."""
-
- def __init__(self, log_manager, monitor, footer):
- self.monitor = monitor
- OutputManager.__init__(self, log_manager, footer)
-
- def __exit__(self, exc_type, exc_value, traceback):
- OutputManager.__exit__(self, exc_type, exc_value, traceback)
-
- # Ensure the resource monitor is stopped because leaving it running
- # could result in the process hanging on exit because the resource
- # collection child process hasn't been told to stop.
- self.monitor.stop_resource_recording()
-
-
- def on_line(self, line):
- warning, state_changed, relevant = self.monitor.on_line(line)
-
- if relevant:
- self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
- elif state_changed:
- have_handler = hasattr(self, 'handler')
- if have_handler:
- self.handler.acquire()
- try:
- self.refresh()
- finally:
- if have_handler:
- self.handler.release()
-
-
class StoreDebugParamsAndWarnAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
sys.stderr.write('The --debugparams argument is deprecated. Please ' +
'use --debugger-args instead.\n\n')
setattr(namespace, self.dest, values)
@CommandProvider
@@ -382,18 +165,21 @@ class Build(MachCommandBase):
libraries and executables (binaries).
* faster - builds JavaScript, XUL, CSS, etc files.
"binaries" and "faster" almost fully complement each other. However,
there are build actions not captured by either. If things don't appear to
be rebuilding, perform a vanilla `mach build` to rebuild the world.
"""
- import which
- from mozbuild.controller.building import BuildMonitor
+ from mozbuild.controller.building import (
+ BuildMonitor,
+ BuildOutputManager,
+ BuildProgressFooter,
+ )
from mozbuild.util import (
mkdir,
resolve_target_to_make,
)
self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
@@ -2116,72 +1902,16 @@ class StaticAnalysisMonitor(object):
self._current = os.path.relpath(filename, self._srcdir)
else:
self._current = None
self._processed = self._processed + 1
return (warning, False)
return (warning, True)
-class StaticAnalysisFooter(Footer):
- """Handles display of a static analysis progress indicator in a terminal.
- """
-
- def __init__(self, terminal, monitor):
- Footer.__init__(self, terminal)
- self.monitor = monitor
-
- def draw(self):
- """Draws this footer in the terminal."""
-
- monitor = self.monitor
- total = monitor.num_files
- processed = monitor.num_files_processed
- percent = '(%.2f%%)' % (processed * 100.0 / total)
- parts = [
- ('dim', 'Processing'),
- ('yellow', str(processed)),
- ('dim', 'of'),
- ('yellow', str(total)),
- ('dim', 'files'),
- ('green', percent)
- ]
- if monitor.current_file:
- parts.append(('bold', monitor.current_file))
-
- self.write(parts)
-
-
-class StaticAnalysisOutputManager(OutputManager):
- """Handles writing static analysis output to a terminal."""
-
- def __init__(self, log_manager, monitor, footer):
- self.monitor = monitor
- OutputManager.__init__(self, log_manager, footer)
-
- def on_line(self, line):
- warning, relevant = self.monitor.on_line(line)
-
- if warning:
- self.log(logging.INFO, 'compiler_warning', warning,
- 'Warning: {flag} in {filename}: {message}')
-
- if relevant:
- self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
- else:
- have_handler = hasattr(self, 'handler')
- if have_handler:
- self.handler.acquire()
- try:
- self.refresh()
- finally:
- if have_handler:
- self.handler.release()
-
-
@CommandProvider
class StaticAnalysis(MachCommandBase):
"""Utilities for running C++ static analysis checks."""
@Command('static-analysis', category='testing',
description='Run C++ static analysis checks')
def static_analysis(self):
# If not arguments are provided, just print a help message.
@@ -2208,16 +1938,21 @@ class StaticAnalysis(MachCommandBase):
@CommandArgument('--fix', '-f', default=False, action='store_true',
help='Try to autofix errors detected by clang-tidy checkers.')
@CommandArgument('--header-filter', '-h-f', default='', metavar='header_filter',
help='Regular expression matching the names of the headers to '
'output diagnostics from. Diagnostics from the main file '
'of each translation unit are always displayed')
def check(self, source=None, jobs=2, strip=1, verbose=False,
checks='-*', fix=False, header_filter=''):
+ from mozbuild.controller.building import (
+ StaticAnalysisFooter,
+ StaticAnalysisOutputManager,
+ )
+
self._set_log_level(verbose)
rc = self._build_compile_db(verbose=verbose)
if rc != 0:
return rc
rc = self._build_export(jobs=jobs, verbose=verbose)
if rc != 0:
return rc