Bug 1411994 - Ignore Unicode errors when logging; r?glandium draft
authorGregory Szorc <gps@mozilla.com>
Thu, 26 Oct 2017 15:27:01 -0700
changeset 689492 eedc3b8d6c449a8d7f782f35878a85b14f04c44f
parent 689459 0a7ff6e19bcc229500d92597fac9340d9bdef959
child 738326 8681574e4ae5920fc802270f9433d527f119d161
push id87035
push usergszorc@mozilla.com
push dateTue, 31 Oct 2017 16:20:22 +0000
reviewersglandium
bugs1411994
milestone58.0a1
Bug 1411994 - Ignore Unicode errors when logging; r?glandium LineIO currently attempts to convert bytes to unicode using the system preferred encoding. This can obviously fail for some byte sequences. In many cases, the conversion is happening for purposes of logging. In these cases, Unicode validation shouldn't be that important to us. So, this commit teaches LineIO to accept an argument defining its Unicode error mode. We also teach relevant users of LineIO doing logging to switch the default mode from "strict" to "replace" so invalid Unicode byte sequences can be logged with the Unicode replacement character. During review, glandium pointed out that it may make better sense to defer Unicode decoding to the logging layer instead of in LineIO. I agree with this idea. But it can be implemented in another bug: for now we should unbust the build system. MozReview-Commit-ID: 7miuSDY8Xzv
build/moz.configure/init.configure
build/moz.configure/pkg.configure
build/moz.configure/util.configure
python/mozbuild/mozbuild/configure/util.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -186,17 +186,17 @@ def virtualenv_python(env_python, build_
             python = mozconfig['vars']['modified']['PYTHON'][1]
 
     with LineIO(lambda l: log.error(l)) as out:
         verify_python_version(out)
     topsrcdir, topobjdir = build_env.topsrcdir, build_env.topobjdir
     if topobjdir.endswith('/js/src'):
         topobjdir = topobjdir[:-7]
 
-    with LineIO(lambda l: log.info(l)) as out:
+    with LineIO(lambda l: log.info(l), 'replace') as out:
         manager = VirtualenvManager(
             topsrcdir, topobjdir,
             os.path.join(topobjdir, '_virtualenv'), out,
             os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
 
     if python:
         # If we're not in the virtualenv, we need the which module for
         # find_program.
--- a/build/moz.configure/pkg.configure
+++ b/build/moz.configure/pkg.configure
@@ -69,17 +69,17 @@ def pkg_check_modules(var, package_desc,
             try:
                 subprocess.check_output([pkg_config, '--errors-to-stdout',
                                          '--print-errors', package_desc])
                 log.info("yes")
                 return True
             except subprocess.CalledProcessError as e:
                 log.info("no")
                 log_writer = log.warning if allow_missing else log.error
-                with LineIO(lambda l: log_writer(l)) as o:
+                with LineIO(lambda l: log_writer(l), 'replace') as o:
                     o.write(e.output)
                 if not allow_missing:
                     sys.exit(1)
 
     @depends(pkg_config, package_desc, when=package)
     @checking('%s_CFLAGS' % var, callback=lambda t: ' '.join(t))
     def pkg_cflags(pkg_config, package_desc):
         flags = check_cmd_output(pkg_config, '--cflags', package_desc)
--- a/build/moz.configure/util.configure
+++ b/build/moz.configure/util.configure
@@ -58,17 +58,17 @@ def check_cmd_output(*args, **kwargs):
         if retcode == 0:
             return stdout
 
         log.debug('The command returned non-zero exit status %d.',
                   retcode)
         for out, desc in ((stdout, 'output'), (stderr, 'error output')):
             if out:
                 log.debug('Its %s was:', desc)
-                with LineIO(lambda l: log.debug('| %s', l)) as o:
+                with LineIO(lambda l: log.debug('| %s', l), 'replace') as o:
                     o.write(out)
         if onerror:
             return onerror()
         die('Command `%s` failed with exit status %d.' %
             (quote(*args), retcode))
 
 
 @imports('os')
--- a/python/mozbuild/mozbuild/configure/util.py
+++ b/python/mozbuild/mozbuild/configure/util.py
@@ -189,24 +189,25 @@ class ConfigureOutputHandler(logging.Han
                 break
         self._keep_if_debug = self.KEEP
 
 
 class LineIO(object):
     '''File-like class that sends each line of the written data to a callback
     (without carriage returns).
     '''
-    def __init__(self, callback):
+    def __init__(self, callback, errors='strict'):
         self._callback = callback
         self._buf = ''
         self._encoding = getpreferredencoding()
+        self._errors = errors
 
     def write(self, buf):
         if self._encoding and isinstance(buf, str):
-            buf = buf.decode(self._encoding)
+            buf = buf.decode(self._encoding, self._errors)
         lines = buf.splitlines()
         if not lines:
             return
         if self._buf:
             lines[0] = self._buf + lines[0]
             self._buf = ''
         if not buf.endswith('\n'):
             self._buf = lines.pop()