Bug 1411994 - Ignore Unicode errors when logging; r?glandium
LineIO currently attempts to convert bytes to unicode using the
system preferred encoding. This can obviously fail for some byte
sequences.
In many cases, the conversion is happening for purposes of logging.
In these cases, Unicode validation shouldn't be that important to
us.
So, this commit teaches LineIO to accept an argument defining its
Unicode error mode. We also teach relevant users of LineIO doing
logging to switch the default mode from "strict" to "replace" so
invalid Unicode byte sequences can be logged with the Unicode
replacement character.
During review, glandium pointed out that it may make better sense
to defer Unicode decoding to the logging layer instead of in LineIO.
I agree with this idea. But it can be implemented in another bug:
for now we should unbust the build system.
MozReview-Commit-ID: 7miuSDY8Xzv
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -186,17 +186,17 @@ def virtualenv_python(env_python, build_
python = mozconfig['vars']['modified']['PYTHON'][1]
with LineIO(lambda l: log.error(l)) as out:
verify_python_version(out)
topsrcdir, topobjdir = build_env.topsrcdir, build_env.topobjdir
if topobjdir.endswith('/js/src'):
topobjdir = topobjdir[:-7]
- with LineIO(lambda l: log.info(l)) as out:
+ with LineIO(lambda l: log.info(l), 'replace') as out:
manager = VirtualenvManager(
topsrcdir, topobjdir,
os.path.join(topobjdir, '_virtualenv'), out,
os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
if python:
# If we're not in the virtualenv, we need the which module for
# find_program.
--- a/build/moz.configure/pkg.configure
+++ b/build/moz.configure/pkg.configure
@@ -69,17 +69,17 @@ def pkg_check_modules(var, package_desc,
try:
subprocess.check_output([pkg_config, '--errors-to-stdout',
'--print-errors', package_desc])
log.info("yes")
return True
except subprocess.CalledProcessError as e:
log.info("no")
log_writer = log.warning if allow_missing else log.error
- with LineIO(lambda l: log_writer(l)) as o:
+ with LineIO(lambda l: log_writer(l), 'replace') as o:
o.write(e.output)
if not allow_missing:
sys.exit(1)
@depends(pkg_config, package_desc, when=package)
@checking('%s_CFLAGS' % var, callback=lambda t: ' '.join(t))
def pkg_cflags(pkg_config, package_desc):
flags = check_cmd_output(pkg_config, '--cflags', package_desc)
--- a/build/moz.configure/util.configure
+++ b/build/moz.configure/util.configure
@@ -58,17 +58,17 @@ def check_cmd_output(*args, **kwargs):
if retcode == 0:
return stdout
log.debug('The command returned non-zero exit status %d.',
retcode)
for out, desc in ((stdout, 'output'), (stderr, 'error output')):
if out:
log.debug('Its %s was:', desc)
- with LineIO(lambda l: log.debug('| %s', l)) as o:
+ with LineIO(lambda l: log.debug('| %s', l), 'replace') as o:
o.write(out)
if onerror:
return onerror()
die('Command `%s` failed with exit status %d.' %
(quote(*args), retcode))
@imports('os')
--- a/python/mozbuild/mozbuild/configure/util.py
+++ b/python/mozbuild/mozbuild/configure/util.py
@@ -189,24 +189,25 @@ class ConfigureOutputHandler(logging.Han
break
self._keep_if_debug = self.KEEP
class LineIO(object):
'''File-like class that sends each line of the written data to a callback
(without carriage returns).
'''
- def __init__(self, callback):
+ def __init__(self, callback, errors='strict'):
self._callback = callback
self._buf = ''
self._encoding = getpreferredencoding()
+ self._errors = errors
def write(self, buf):
if self._encoding and isinstance(buf, str):
- buf = buf.decode(self._encoding)
+ buf = buf.decode(self._encoding, self._errors)
lines = buf.splitlines()
if not lines:
return
if self._buf:
lines[0] = self._buf + lines[0]
self._buf = ''
if not buf.endswith('\n'):
self._buf = lines.pop()