Bug 1391019 - Add py2 and py3 compatability linters, r?gps draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 31 Aug 2017 10:12:02 -0400
changeset 656852 53bfeaad613b9975f8b86bc7f7824bf343cb0677
parent 656680 90922503ef351dece72f9e3c876c7eb49bd16a33
child 656853 ee4522d792ab129194adcf9e9e46a0f24f1f9804
push id77342
push userahalberstadt@mozilla.com
push dateThu, 31 Aug 2017 19:31:55 +0000
reviewersgps
bugs1391019
milestone57.0a1
Bug 1391019 - Add py2 and py3 compatability linters, r?gps check_compat.py was adapted from gps' check-py3-compat.py in mercurial: https://www.mercurial-scm.org/repo/hg/file/tip/contrib/check-py3-compat.py The py3 linter simply runs ast.parse(f) for each file being linted. Any syntax errors are formatted as mozlint results and dumped to stdout as json. I looked into also importing the file (using 3.5+'s importlib.util.spec_from_file_location), but there were too many problems: 1. Lots of false positives (e.g module not found) 2. Some files seemed to run indefinitely on import I decided to punt on importing for now, we can always investigate in a follow-up. The py2 linter runs ast.parse(f), and also checks that the file has: from __future__ import absolute_import, print_function Initially every python file in the tree is excluded from the py2 check, though at least this makes it easy to find+fix, and new files in un-excluded directories will automatically be linted. MozReview-Commit-ID: ABtq9dnPo9T
tools/lint/py2.yml
tools/lint/py3.yml
tools/lint/python/check_compat.py
tools/lint/python/compat.py
new file mode 100644
--- /dev/null
+++ b/tools/lint/py2.yml
@@ -0,0 +1,75 @@
+---
+py2:
+    description: Python 2 compatibility check
+    include: ['.']
+    exclude:
+        - accessible/xpcom/AccEventGen.py
+        - addon-sdk
+        - browser
+        - build
+        - client.py
+        - config
+        - configure.py
+        - devtools/shared/css/generated/mach_commands.py
+        - dom
+        - editor
+        - gfx
+        - intl
+        - ipc
+        - js/src
+        - js/xpconnect
+        - layout
+        - media
+        - memory
+        - mobile
+        - modules
+        - mozglue
+        - netwerk
+        - nsprpub
+        - other-licenses
+        - probes/trace-gen.py
+        - python/devtools
+        - python/mach
+        - python/mozboot
+        - python/mozbuild
+        - python/mozlint
+        - python/mozversioncontrol
+        - security
+        - services/common/tests/mach_commands.py
+        - servo
+        - taskcluster/docker
+        - taskcluster/taskgraph
+        - testing/awsy
+        - testing/firefox-ui
+        - testing/geckodriver
+        - testing/gtest
+        - testing/instrumentation/runinstrumentation.py
+        - testing/marionette
+        - testing/mochitest
+        - testing/mozbase
+        - testing/mozharness
+        - testing/remotecppunittests.py
+        - testing/runcppunittests.py
+        - testing/runtimes
+        - testing/talos
+        - testing/tools
+        - testing/tps
+        - testing/web-platform
+        - testing/xpcshell
+        - third_party
+        - toolkit
+        - tools/docs
+        - tools/git/eslintvalidate.py
+        - tools/jprof/split-profile.py
+        - tools/lint
+        - tools/mach_commands.py
+        - tools/mercurial/eslintvalidate.py
+        - tools/power/mach_commands.py
+        - tools/profiler
+        - tools/rb
+        - tools/tryselect
+        - tools/update-packaging
+        - xpcom
+    extensions: ['py']
+    type: external
+    payload: python.compat:lintpy2
new file mode 100644
--- /dev/null
+++ b/tools/lint/py3.yml
@@ -0,0 +1,55 @@
+---
+py3:
+    description: Python 3 compatibility check
+    include: ['.']
+    exclude:
+        - addon-sdk/source
+        - browser/app
+        - browser/components
+        - browser/extensions
+        - build
+        - client.py
+        - config
+        - dom/bindings
+        - dom/canvas/test
+        - dom/media/test
+        - gfx
+        - intl/icu
+        - ipc/chromium
+        - ipc/ipdl
+        - js/src
+        - layout/reftests
+        - layout/style
+        - layout/tools/reftest
+        - media
+        - memory/replace
+        - modules/freetype2
+        - nsprpub
+        - security/manager/ssl
+        - security/nss
+        - services/common/tests/mach_commands.py
+        - servo
+        - testing/awsy
+        - testing/firefox-ui/harness/firefox_ui_harness/runners/update.py
+        - testing/gtest
+        - testing/marionette
+        - testing/mochitest
+        - testing/mozbase
+        - testing/mozharness
+        - testing/talos
+        - testing/tools/iceserver
+        - testing/tps
+        - testing/xpcshell
+        - testing/web-platform
+        - third_party
+        - toolkit
+        - tools/git
+        - tools/jprof
+        - tools/profiler
+        - tools/rb
+        - tools/update-packaging
+        - xpcom/idl-parser
+        - xpcom/typelib
+    extensions: ['py']
+    type: external
+    payload: python.compat:lintpy3
new file mode 100755
--- /dev/null
+++ b/tools/lint/python/check_compat.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import ast
+import json
+import sys
+
+
+def parse_file(f):
+    with open(f, 'rb') as fh:
+        content = fh.read()
+    try:
+        return ast.parse(content)
+    except SyntaxError as e:
+        err = {
+            'path': f,
+            'message': e.msg,
+            'lineno': e.lineno,
+            'column': e.offset,
+            'source': e.text,
+            'rule': 'is-parseable',
+        }
+        print(json.dumps(err))
+
+
+def check_compat_py2(f):
+    """Check Python 2 and Python 3 compatibility for a file with Python 2"""
+    root = parse_file(f)
+
+    # Ignore empty or un-parseable files.
+    if not root or not root.body:
+        return
+
+    futures = set()
+    haveprint = False
+    future_lineno = 1
+    for node in ast.walk(root):
+        if isinstance(node, ast.ImportFrom):
+            if node.module == '__future__':
+                future_lineno = node.lineno
+                futures |= set(n.name for n in node.names)
+        elif isinstance(node, ast.Print):
+            haveprint = True
+
+    err = {
+        'path': f,
+        'lineno': future_lineno,
+        'column': 1,
+    }
+
+    if 'absolute_import' not in futures:
+        err['rule'] = 'require absolute_import'
+        err['message'] = 'Missing from __future__ import absolute_import'
+        print(json.dumps(err))
+
+    if haveprint and 'print_function' not in futures:
+        err['rule'] = 'require print_function'
+        err['message'] = 'Missing from __future__ import print_function'
+        print(json.dumps(err))
+
+
+def check_compat_py3(f):
+    """Check Python 3 compatibility of a file with Python 3."""
+    parse_file(f)
+
+
+if __name__ == '__main__':
+    if sys.version_info[0] == 2:
+        fn = check_compat_py2
+    else:
+        fn = check_compat_py3
+
+    manifest = sys.argv[1]
+    with open(manifest, 'r') as fh:
+        files = fh.read().splitlines()
+
+    for f in files:
+        fn(f)
+
+    sys.exit(0)
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/compat.py
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import json
+import os
+import tempfile
+from distutils.spawn import find_executable
+
+from mozpack.files import FileFinder
+from mozprocess import ProcessHandlerMixin
+
+from mozlint import result
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+results = []
+
+
+class PyCompatProcess(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
+
+    def process_line(self, line):
+        try:
+            res = json.loads(line)
+        except ValueError:
+            print('Non JSON output from linter, will not be processed: {}'.format(line))
+            return
+
+        res['level'] = 'error'
+        results.append(result.from_config(self.config, **res))
+
+
+def run_linter(python, paths, config, **lintargs):
+    binary = find_executable(python)
+    if not binary:
+        # TODO bootstrap python3 if not available
+        print('error: {} not detected, aborting py-compat check'.format(python))
+        if 'MOZ_AUTOMATION' in os.environ:
+            return 1
+        return []
+
+    pattern = "**/*.py"
+    exclude = lintargs.get('exclude', [])
+    files = []
+    for path in paths:
+        if os.path.isfile(path):
+            files.append(path)
+            continue
+
+        finder = FileFinder(path, ignore=exclude)
+        files.extend([os.path.join(path, p) for p, f in finder.find(pattern)])
+
+    with tempfile.NamedTemporaryFile(mode='w') as fh:
+        fh.write('\n'.join(files))
+        fh.flush()
+
+        cmd = [binary, os.path.join(here, 'check_compat.py'), fh.name]
+
+        proc = PyCompatProcess(config, cmd)
+        proc.run()
+        try:
+            proc.wait()
+        except KeyboardInterrupt:
+            proc.kill()
+
+    return results
+
+
+def lintpy2(*args, **kwargs):
+    return run_linter('python2', *args, **kwargs)
+
+
+def lintpy3(*args, **kwargs):
+    return run_linter('python3', *args, **kwargs)