Bug 1391019 - Add py2 and py3 compatability linters, r?gps
check_compat.py was adapted from gps' check-py3-compat.py in mercurial:
https://www.mercurial-scm.org/repo/hg/file/tip/contrib/check-py3-compat.py
The py3 linter simply runs ast.parse(f) for each file being linted. Any syntax errors
are formatted as mozlint results and dumped to stdout as json. I looked into also
importing the file (using 3.5+'s importlib.util.spec_from_file_location), but there
were too many problems:
1. Lots of false positives (e.g module not found)
2. Some files seemed to run indefinitely on import
I decided to punt on importing for now, we can always investigate in a follow-up.
The py2 linter runs ast.parse(f), and also checks that the file has:
from __future__ import absolute_import, print_function
Initially every python file in the tree is excluded from the py2 check, though
at least this makes it easy to find+fix, and new files in un-excluded
directories will automatically be linted.
MozReview-Commit-ID: ABtq9dnPo9T
new file mode 100644
--- /dev/null
+++ b/tools/lint/py2.yml
@@ -0,0 +1,75 @@
+---
+py2:
+ description: Python 2 compatibility check
+ include: ['.']
+ exclude:
+ - accessible/xpcom/AccEventGen.py
+ - addon-sdk
+ - browser
+ - build
+ - client.py
+ - config
+ - configure.py
+ - devtools/shared/css/generated/mach_commands.py
+ - dom
+ - editor
+ - gfx
+ - intl
+ - ipc
+ - js/src
+ - js/xpconnect
+ - layout
+ - media
+ - memory
+ - mobile
+ - modules
+ - mozglue
+ - netwerk
+ - nsprpub
+ - other-licenses
+ - probes/trace-gen.py
+ - python/devtools
+ - python/mach
+ - python/mozboot
+ - python/mozbuild
+ - python/mozlint
+ - python/mozversioncontrol
+ - security
+ - services/common/tests/mach_commands.py
+ - servo
+ - taskcluster/docker
+ - taskcluster/taskgraph
+ - testing/awsy
+ - testing/firefox-ui
+ - testing/geckodriver
+ - testing/gtest
+ - testing/instrumentation/runinstrumentation.py
+ - testing/marionette
+ - testing/mochitest
+ - testing/mozbase
+ - testing/mozharness
+ - testing/remotecppunittests.py
+ - testing/runcppunittests.py
+ - testing/runtimes
+ - testing/talos
+ - testing/tools
+ - testing/tps
+ - testing/web-platform
+ - testing/xpcshell
+ - third_party
+ - toolkit
+ - tools/docs
+ - tools/git/eslintvalidate.py
+ - tools/jprof/split-profile.py
+ - tools/lint
+ - tools/mach_commands.py
+ - tools/mercurial/eslintvalidate.py
+ - tools/power/mach_commands.py
+ - tools/profiler
+ - tools/rb
+ - tools/tryselect
+ - tools/update-packaging
+ - xpcom
+ extensions: ['py']
+ type: external
+ payload: python.compat:lintpy2
new file mode 100644
--- /dev/null
+++ b/tools/lint/py3.yml
@@ -0,0 +1,55 @@
+---
+py3:
+ description: Python 3 compatibility check
+ include: ['.']
+ exclude:
+ - addon-sdk/source
+ - browser/app
+ - browser/components
+ - browser/extensions
+ - build
+ - client.py
+ - config
+ - dom/bindings
+ - dom/canvas/test
+ - dom/media/test
+ - gfx
+ - intl/icu
+ - ipc/chromium
+ - ipc/ipdl
+ - js/src
+ - layout/reftests
+ - layout/style
+ - layout/tools/reftest
+ - media
+ - memory/replace
+ - modules/freetype2
+ - nsprpub
+ - security/manager/ssl
+ - security/nss
+ - services/common/tests/mach_commands.py
+ - servo
+ - testing/awsy
+ - testing/firefox-ui/harness/firefox_ui_harness/runners/update.py
+ - testing/gtest
+ - testing/marionette
+ - testing/mochitest
+ - testing/mozbase
+ - testing/mozharness
+ - testing/talos
+ - testing/tools/iceserver
+ - testing/tps
+ - testing/xpcshell
+ - testing/web-platform
+ - third_party
+ - toolkit
+ - tools/git
+ - tools/jprof
+ - tools/profiler
+ - tools/rb
+ - tools/update-packaging
+ - xpcom/idl-parser
+ - xpcom/typelib
+ extensions: ['py']
+ type: external
+ payload: python.compat:lintpy3
new file mode 100755
--- /dev/null
+++ b/tools/lint/python/check_compat.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import ast
+import json
+import sys
+
+
+def parse_file(f):
+ with open(f, 'rb') as fh:
+ content = fh.read()
+ try:
+ return ast.parse(content)
+ except SyntaxError as e:
+ err = {
+ 'path': f,
+ 'message': e.msg,
+ 'lineno': e.lineno,
+ 'column': e.offset,
+ 'source': e.text,
+ 'rule': 'is-parseable',
+ }
+ print(json.dumps(err))
+
+
+def check_compat_py2(f):
+ """Check Python 2 and Python 3 compatibility for a file with Python 2"""
+ root = parse_file(f)
+
+ # Ignore empty or un-parseable files.
+ if not root or not root.body:
+ return
+
+ futures = set()
+ haveprint = False
+ future_lineno = 1
+ for node in ast.walk(root):
+ if isinstance(node, ast.ImportFrom):
+ if node.module == '__future__':
+ future_lineno = node.lineno
+ futures |= set(n.name for n in node.names)
+ elif isinstance(node, ast.Print):
+ haveprint = True
+
+ err = {
+ 'path': f,
+ 'lineno': future_lineno,
+ 'column': 1,
+ }
+
+ if 'absolute_import' not in futures:
+ err['rule'] = 'require absolute_import'
+ err['message'] = 'Missing from __future__ import absolute_import'
+ print(json.dumps(err))
+
+ if haveprint and 'print_function' not in futures:
+ err['rule'] = 'require print_function'
+ err['message'] = 'Missing from __future__ import print_function'
+ print(json.dumps(err))
+
+
+def check_compat_py3(f):
+ """Check Python 3 compatibility of a file with Python 3."""
+ parse_file(f)
+
+
+if __name__ == '__main__':
+ if sys.version_info[0] == 2:
+ fn = check_compat_py2
+ else:
+ fn = check_compat_py3
+
+ manifest = sys.argv[1]
+ with open(manifest, 'r') as fh:
+ files = fh.read().splitlines()
+
+ for f in files:
+ fn(f)
+
+ sys.exit(0)
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/compat.py
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import json
+import os
+import tempfile
+from distutils.spawn import find_executable
+
+from mozpack.files import FileFinder
+from mozprocess import ProcessHandlerMixin
+
+from mozlint import result
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+results = []
+
+
+class PyCompatProcess(ProcessHandlerMixin):
+ def __init__(self, config, *args, **kwargs):
+ self.config = config
+ kwargs['processOutputLine'] = [self.process_line]
+ ProcessHandlerMixin.__init__(self, *args, **kwargs)
+
+ def process_line(self, line):
+ try:
+ res = json.loads(line)
+ except ValueError:
+ print('Non JSON output from linter, will not be processed: {}'.format(line))
+ return
+
+ res['level'] = 'error'
+ results.append(result.from_config(self.config, **res))
+
+
+def run_linter(python, paths, config, **lintargs):
+ binary = find_executable(python)
+ if not binary:
+ # TODO bootstrap python3 if not available
+ print('error: {} not detected, aborting py-compat check'.format(python))
+ if 'MOZ_AUTOMATION' in os.environ:
+ return 1
+ return []
+
+ pattern = "**/*.py"
+ exclude = lintargs.get('exclude', [])
+ files = []
+ for path in paths:
+ if os.path.isfile(path):
+ files.append(path)
+ continue
+
+ finder = FileFinder(path, ignore=exclude)
+ files.extend([os.path.join(path, p) for p, f in finder.find(pattern)])
+
+ with tempfile.NamedTemporaryFile(mode='w') as fh:
+ fh.write('\n'.join(files))
+ fh.flush()
+
+ cmd = [binary, os.path.join(here, 'check_compat.py'), fh.name]
+
+ proc = PyCompatProcess(config, cmd)
+ proc.run()
+ try:
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+
+ return results
+
+
+def lintpy2(*args, **kwargs):
+ return run_linter('python2', *args, **kwargs)
+
+
+def lintpy3(*args, **kwargs):
+ return run_linter('python3', *args, **kwargs)