Bug 1270506 - [mozlint] Add python flake8 linter, r?smacleod draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 05 May 2016 17:21:12 -0400
changeset 367830 0e6dbc3c06e64aaa206f52af29bb67dd9b12d801
parent 367829 195fbc63b501412d8e3a009efbcbd74b3112b232
child 367889 d49b4598fb5e1ed60ef1005a25326db4a4355433
child 367949 69b886215f3a2a741bb546a527256bc3bf472b10
child 367953 63d856a6261b19c911c098dba6ef81d6e9d06d02
push id18366
push userahalberstadt@mozilla.com
push dateTue, 17 May 2016 13:04:02 +0000
reviewerssmacleod
bugs1270506
milestone49.0a1
Bug 1270506 - [mozlint] Add python flake8 linter, r?smacleod For now, only the following two directories will be linted: python/mozlint tools/lint New directories can be added by adding them to the 'include' directive in tools/lint/flake8.lint. They all default to the configuration specified in topsrcdir/.flake8. Subdirectories can override this configuration by creating their own .flake8 file. MozReview-Commit-ID: Eag48Lnkp3l
.flake8
python/mozlint/mozlint/result.py
python/mozlint/mozlint/roller.py
python/mozlint/mozlint/types.py
python/mozlint/test/linters/regex.lint
python/mozlint/test/linters/string.lint
tools/lint/docs/conf.py
tools/lint/docs/index.rst
tools/lint/docs/linters/flake8.rst
tools/lint/flake8.lint
tools/lint/mach_commands.py
new file mode 100644
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 99
+filename = *.py, *.lint
--- a/python/mozlint/mozlint/result.py
+++ b/python/mozlint/mozlint/result.py
@@ -29,25 +29,25 @@ class ResultContainer(object):
         'column',
         'hint',
         'source',
         'level',
         'rule',
         'lineoffset',
     )
 
-    def __init__(self, linter, path, message, lineno, column=1, hint=None,
-                 source=None, level='error', rule=None, lineoffset=None):
+    def __init__(self, linter, path, message, lineno, column=None, hint=None,
+                 source=None, level=None, rule=None, lineoffset=None):
         self.path = path
         self.message = message
         self.lineno = lineno
-        self.column = column
+        self.column = column or 1
         self.hint = hint
         self.source = source
-        self.level = level
+        self.level = level or 'error'
         self.linter = linter
         self.rule = rule
         self.lineoffset = lineoffset
 
     def __repr__(self):
         s = dumps(self, cls=ResultEncoder, indent=2)
         return "ResultContainer({})".format(s)
 
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -1,12 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import unicode_literals
+
 import signal
 import traceback
 from collections import defaultdict
 from Queue import Empty
 from multiprocessing import (
     Manager,
     Pool,
     cpu_count,
@@ -30,16 +32,19 @@ def _run_linters(queue, paths, **lintarg
         # Ideally we would pass the entire LINTER definition as an argument
         # to the worker instead of re-parsing it. But passing a function from
         # a dynamically created module (with imp) does not seem to be possible
         # with multiprocessing on Windows.
         linter = parse(linter_path)
         func = supported_types[linter['type']]
         res = func(paths, linter, **lintargs) or []
 
+        if isinstance(res, basestring):
+            continue
+
         for r in res:
             results[r.path].append(r)
 
 
 def _run_worker(*args, **lintargs):
     try:
         return _run_linters(*args, **lintargs)
     except:
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -25,16 +25,17 @@ class BaseType(object):
                          the definition, but passed in by a consumer.
         :returns: A list of :class:`~result.ResultContainer` objects.
         """
         exclude = lintargs.get('exclude', [])
         exclude.extend(linter.get('exclude', []))
 
         paths, exclude = filterpaths(paths, linter.get('include'), exclude)
         if not paths:
+            print("{}: No files to lint for specified paths!".format(linter['name']))
             return
 
         lintargs['exclude'] = exclude
         if self.batch:
             return self._lint(paths, linter, **lintargs)
 
         errors = []
         for p in paths:
--- a/python/mozlint/test/linters/regex.lint
+++ b/python/mozlint/test/linters/regex.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "RegexLinter",
-    'description': "Make sure the string 'foobar' never appears in a js variable files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in a js variable file because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'regex',
     'payload': 'foobar',
 }
--- a/python/mozlint/test/linters/string.lint
+++ b/python/mozlint/test/linters/string.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "StringLinter",
-    'description': "Make sure the string 'foobar' never appears in browser js files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in browser js files because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'string',
     'payload': 'foobar',
 }
--- a/tools/lint/docs/conf.py
+++ b/tools/lint/docs/conf.py
@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 #
 # mozlint documentation build configuration file, created by
 # sphinx-quickstart on Fri Nov 27 17:38:49 2015.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
 
-import sys
 import os
-import shlex
 
 # -- General configuration ------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
--- a/tools/lint/docs/index.rst
+++ b/tools/lint/docs/index.rst
@@ -21,15 +21,16 @@ 2. It provides a streamlined interface f
 like mach, mozreview and taskcluster.
 
 .. toctree::
   :caption: Linting User Guide
   :maxdepth: 2
 
   usage
   create
+  linters/flake8
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
new file mode 100644
--- /dev/null
+++ b/tools/lint/docs/linters/flake8.rst
@@ -0,0 +1,44 @@
+Flake8
+======
+
+`Flake8`_ is a popular lint wrapper for python. Under the hood, it runs three other tools and
+combines their results:
+
+* `pep8`_ for checking style
+* `pyflakes`_ for checking syntax
+* `mccabe`_ for checking complexity
+
+
+Run Locally
+-----------
+
+The mozlint integration of flake8 can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter flake8 <file paths>
+
+Alternatively, omit the ``--linter flake8`` and run all configured linters, which will include
+flake8.
+
+
+Configuration
+-------------
+
+Only directories explicitly whitelisted will have flake8 run against them. To enable flake8 linting
+in a source directory, it must be added to the ``include`` directive in ```tools/lint/flake8.lint``.
+If you wish to exclude a subdirectory of an included one, you can add it to the ``exclude``
+directive.
+
+The default configuration file lives in ``topsrcdir/.flake8``. The default configuration can be
+overriden for a given subdirectory by creating a new ``.flake8`` file in the subdirectory. Be warned
+that ``.flake8`` files cannot inherit from one another, so all configuration you wish to keep must
+be re-defined.
+
+For an overview of the supported configuration, see `flake8's documentation`_.
+
+.. _Flake8: https://flake8.readthedocs.io/en/latest/
+.. _pep8: http://pep8.readthedocs.io/en/latest/
+.. _pyflakes: https://github.com/pyflakes/pyflakes
+.. _mccabe: https://github.com/pycqa/mccabe
+.. _flake8's documentation: https://flake8.readthedocs.io/en/latest/config.html
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.lint
@@ -0,0 +1,107 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+
+from mozlint import result
+
+
+FLAKE8_NOT_FOUND = """
+Could not find flake8! Install flake8 and try again.
+
+    $ pip install flake8
+""".strip()
+
+
+LINE_OFFSETS = {
+    # continuation line under-indented for hanging indent
+    'E121': (-1, 2),
+    # continuation line missing indentation or outdented
+    'E122': (-1, 2),
+    # continuation line over-indented for hanging indent
+    'E126': (-1, 2),
+    # continuation line over-indented for visual indent
+    'E127': (-1, 2),
+    # continuation line under-indented for visual indent
+    'E128': (-1, 2),
+    # continuation line unaligned for hanging indend
+    'E131': (-1, 2),
+    # expected 1 blank line, found 0
+    'E301': (-1, 2),
+    # expected 2 blank lines, found 1
+    'E302': (-2, 3),
+}
+"""Maps a flake8 error to a lineoffset tuple.
+
+The offset is of the form (lineno_offset, num_lines) and is passed
+to the lineoffset property of `ResultContainer`.
+"""
+
+
+def lint(files, **lintargs):
+    import which
+
+    binary = os.environ.get('FLAKE8')
+    if not binary:
+        try:
+            binary = which.which('flake8')
+        except which.WhichError:
+            pass
+
+    if not binary:
+        print(FLAKE8_NOT_FOUND)
+        return 1
+
+    cmdargs = [
+        binary,
+        '--format', '{"path":"%(path)s","lineno":%(row)s,'
+                    '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
+    ]
+
+    exclude = lintargs.get('exclude')
+    if exclude:
+        cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
+
+    cmdargs += files
+
+    proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ)
+    output = proc.communicate()[0]
+
+    if not output:
+        return []
+
+    results = []
+    for line in output.splitlines():
+        try:
+            res = json.loads(line)
+        except ValueError:
+            continue
+
+        if 'code' in res:
+            if res['code'].startswith('W'):
+                res['level'] = 'warning'
+
+            if res['code'] in LINE_OFFSETS:
+                res['lineoffset'] = LINE_OFFSETS[res['code']]
+
+        results.append(result.from_linter(LINTER, **res))
+
+    return results
+
+
+LINTER = {
+    'name': "flake8",
+    'description': "Python linter",
+    'include': [
+        'python/mozlint',
+        'tools/lint',
+    ],
+    'exclude': [],
+    'type': 'external',
+    'payload': lint,
+}
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -1,15 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import argparse
 import os
 
 from mozbuild.base import (
     MachCommandBase,
 )
 
 
 from mach.decorators import (
@@ -44,18 +43,18 @@ class MachCommands(MachCommandBase):
     def lint(self, paths, linters=None, fmt='stylish', **lintargs):
         """Run linters."""
         from mozlint import LintRoller, formatters
 
         paths = paths or ['.']
 
         lint_files = self.find_linters(linters)
 
-        lintargs['exclude'] = 'obj*'
-        lint = LintRoller(lintargs=lintargs)
+        lintargs['exclude'] = ['obj*']
+        lint = LintRoller(**lintargs)
         lint.read(lint_files)
 
         # run all linters
         results = lint.roll(paths)
 
         formatter = formatters.get(fmt)
         print(formatter(results))