Bug 1270506 - [mozlint] Add python flake8 linter, r?smacleod
For now, only the following two directories will be linted:
python/mozlint
tools/lint
New directories can be added by adding them to the 'include'
directive in tools/lint/flake8.lint. They all default to the
configuration specified in topsrcdir/.flake8. Subdirectories
can override this configuration by creating their own .flake8
file.
MozReview-Commit-ID: Eag48Lnkp3l
new file mode 100644
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 99
+filename = *.py, *.lint
--- a/python/mozlint/mozlint/result.py
+++ b/python/mozlint/mozlint/result.py
@@ -29,25 +29,25 @@ class ResultContainer(object):
'column',
'hint',
'source',
'level',
'rule',
'lineoffset',
)
- def __init__(self, linter, path, message, lineno, column=1, hint=None,
- source=None, level='error', rule=None, lineoffset=None):
+ def __init__(self, linter, path, message, lineno, column=None, hint=None,
+ source=None, level=None, rule=None, lineoffset=None):
self.path = path
self.message = message
self.lineno = lineno
- self.column = column
+ self.column = column or 1
self.hint = hint
self.source = source
- self.level = level
+ self.level = level or 'error'
self.linter = linter
self.rule = rule
self.lineoffset = lineoffset
def __repr__(self):
s = dumps(self, cls=ResultEncoder, indent=2)
return "ResultContainer({})".format(s)
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -1,12 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import unicode_literals
+
import signal
import traceback
from collections import defaultdict
from Queue import Empty
from multiprocessing import (
Manager,
Pool,
cpu_count,
@@ -30,16 +32,19 @@ def _run_linters(queue, paths, **lintarg
# Ideally we would pass the entire LINTER definition as an argument
# to the worker instead of re-parsing it. But passing a function from
# a dynamically created module (with imp) does not seem to be possible
# with multiprocessing on Windows.
linter = parse(linter_path)
func = supported_types[linter['type']]
res = func(paths, linter, **lintargs) or []
+ if isinstance(res, basestring):
+ continue
+
for r in res:
results[r.path].append(r)
def _run_worker(*args, **lintargs):
try:
return _run_linters(*args, **lintargs)
except:
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -25,16 +25,17 @@ class BaseType(object):
the definition, but passed in by a consumer.
:returns: A list of :class:`~result.ResultContainer` objects.
"""
exclude = lintargs.get('exclude', [])
exclude.extend(linter.get('exclude', []))
paths, exclude = filterpaths(paths, linter.get('include'), exclude)
if not paths:
+ print("{}: No files to lint for specified paths!".format(linter['name']))
return
lintargs['exclude'] = exclude
if self.batch:
return self._lint(paths, linter, **lintargs)
errors = []
for p in paths:
--- a/python/mozlint/test/linters/regex.lint
+++ b/python/mozlint/test/linters/regex.lint
@@ -1,14 +1,15 @@
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
LINTER = {
'name': "RegexLinter",
- 'description': "Make sure the string 'foobar' never appears in a js variable files because it is bad.",
+ 'description': "Make sure the string 'foobar' never appears "
+ "in a js variable file because it is bad.",
'rule': 'no-foobar',
'include': [
'**/*.js',
'**/*.jsm',
],
'type': 'regex',
'payload': 'foobar',
}
--- a/python/mozlint/test/linters/string.lint
+++ b/python/mozlint/test/linters/string.lint
@@ -1,14 +1,15 @@
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
LINTER = {
'name': "StringLinter",
- 'description': "Make sure the string 'foobar' never appears in browser js files because it is bad.",
+ 'description': "Make sure the string 'foobar' never appears "
+ "in browser js files because it is bad.",
'rule': 'no-foobar',
'include': [
'**/*.js',
'**/*.jsm',
],
'type': 'string',
'payload': 'foobar',
}
--- a/tools/lint/docs/conf.py
+++ b/tools/lint/docs/conf.py
@@ -1,19 +1,17 @@
# -*- coding: utf-8 -*-
#
# mozlint documentation build configuration file, created by
# sphinx-quickstart on Fri Nov 27 17:38:49 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
-import sys
import os
-import shlex
# -- General configuration ------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
--- a/tools/lint/docs/index.rst
+++ b/tools/lint/docs/index.rst
@@ -21,15 +21,16 @@ 2. It provides a streamlined interface f
like mach, mozreview and taskcluster.
.. toctree::
:caption: Linting User Guide
:maxdepth: 2
usage
create
+ linters/flake8
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
new file mode 100644
--- /dev/null
+++ b/tools/lint/docs/linters/flake8.rst
@@ -0,0 +1,44 @@
+Flake8
+======
+
+`Flake8`_ is a popular lint wrapper for python. Under the hood, it runs three other tools and
+combines their results:
+
+* `pep8`_ for checking style
+* `pyflakes`_ for checking syntax
+* `mccabe`_ for checking complexity
+
+
+Run Locally
+-----------
+
+The mozlint integration of flake8 can be run using mach:
+
+.. parsed-literal::
+
+ $ mach lint --linter flake8 <file paths>
+
+Alternatively, omit the ``--linter flake8`` and run all configured linters, which will include
+flake8.
+
+
+Configuration
+-------------
+
+Only directories explicitly whitelisted will have flake8 run against them. To enable flake8 linting
+in a source directory, it must be added to the ``include`` directive in ```tools/lint/flake8.lint``.
+If you wish to exclude a subdirectory of an included one, you can add it to the ``exclude``
+directive.
+
+The default configuration file lives in ``topsrcdir/.flake8``. The default configuration can be
+overriden for a given subdirectory by creating a new ``.flake8`` file in the subdirectory. Be warned
+that ``.flake8`` files cannot inherit from one another, so all configuration you wish to keep must
+be re-defined.
+
+For an overview of the supported configuration, see `flake8's documentation`_.
+
+.. _Flake8: https://flake8.readthedocs.io/en/latest/
+.. _pep8: http://pep8.readthedocs.io/en/latest/
+.. _pyflakes: https://github.com/pyflakes/pyflakes
+.. _mccabe: https://github.com/pycqa/mccabe
+.. _flake8's documentation: https://flake8.readthedocs.io/en/latest/config.html
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.lint
@@ -0,0 +1,107 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+
+from mozlint import result
+
+
+FLAKE8_NOT_FOUND = """
+Could not find flake8! Install flake8 and try again.
+
+ $ pip install flake8
+""".strip()
+
+
+LINE_OFFSETS = {
+ # continuation line under-indented for hanging indent
+ 'E121': (-1, 2),
+ # continuation line missing indentation or outdented
+ 'E122': (-1, 2),
+ # continuation line over-indented for hanging indent
+ 'E126': (-1, 2),
+ # continuation line over-indented for visual indent
+ 'E127': (-1, 2),
+ # continuation line under-indented for visual indent
+ 'E128': (-1, 2),
+ # continuation line unaligned for hanging indend
+ 'E131': (-1, 2),
+ # expected 1 blank line, found 0
+ 'E301': (-1, 2),
+ # expected 2 blank lines, found 1
+ 'E302': (-2, 3),
+}
+"""Maps a flake8 error to a lineoffset tuple.
+
+The offset is of the form (lineno_offset, num_lines) and is passed
+to the lineoffset property of `ResultContainer`.
+"""
+
+
+def lint(files, **lintargs):
+ import which
+
+ binary = os.environ.get('FLAKE8')
+ if not binary:
+ try:
+ binary = which.which('flake8')
+ except which.WhichError:
+ pass
+
+ if not binary:
+ print(FLAKE8_NOT_FOUND)
+ return 1
+
+ cmdargs = [
+ binary,
+ '--format', '{"path":"%(path)s","lineno":%(row)s,'
+ '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
+ ]
+
+ exclude = lintargs.get('exclude')
+ if exclude:
+ cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
+
+ cmdargs += files
+
+ proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ)
+ output = proc.communicate()[0]
+
+ if not output:
+ return []
+
+ results = []
+ for line in output.splitlines():
+ try:
+ res = json.loads(line)
+ except ValueError:
+ continue
+
+ if 'code' in res:
+ if res['code'].startswith('W'):
+ res['level'] = 'warning'
+
+ if res['code'] in LINE_OFFSETS:
+ res['lineoffset'] = LINE_OFFSETS[res['code']]
+
+ results.append(result.from_linter(LINTER, **res))
+
+ return results
+
+
+LINTER = {
+ 'name': "flake8",
+ 'description': "Python linter",
+ 'include': [
+ 'python/mozlint',
+ 'tools/lint',
+ ],
+ 'exclude': [],
+ 'type': 'external',
+ 'payload': lint,
+}
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -1,15 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
-import argparse
import os
from mozbuild.base import (
MachCommandBase,
)
from mach.decorators import (
@@ -44,18 +43,18 @@ class MachCommands(MachCommandBase):
def lint(self, paths, linters=None, fmt='stylish', **lintargs):
"""Run linters."""
from mozlint import LintRoller, formatters
paths = paths or ['.']
lint_files = self.find_linters(linters)
- lintargs['exclude'] = 'obj*'
- lint = LintRoller(lintargs=lintargs)
+ lintargs['exclude'] = ['obj*']
+ lint = LintRoller(**lintargs)
lint.read(lint_files)
# run all linters
results = lint.roll(paths)
formatter = formatters.get(fmt)
print(formatter(results))