--- a/python/mozlint/mozlint/cli.py
+++ b/python/mozlint/mozlint/cli.py
@@ -79,24 +79,25 @@ class MozlintParser(ArgumentParser):
def find_linters(linters=None):
lints = []
for search_path in SEARCH_PATHS:
if not os.path.isdir(search_path):
continue
+ sys.path.insert(0, search_path)
files = os.listdir(search_path)
for f in files:
name = os.path.basename(f)
- if not name.endswith('.lint.py'):
+ if not name.endswith('.yml'):
continue
- name = name.rsplit('.', 2)[0]
+ name = name.rsplit('.', 1)[0]
if linters and name not in linters:
continue
lints.append(os.path.join(search_path, f))
return lints
--- a/python/mozlint/mozlint/parser.py
+++ b/python/mozlint/mozlint/parser.py
@@ -1,58 +1,32 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import imp
import os
-import sys
-import uuid
+
+import yaml
from .types import supported_types
from .errors import LinterNotFound, LinterParseError
class Parser(object):
- """Reads and validates `.lint.py` files."""
+ """Reads and validates lint configuration files."""
required_attributes = (
'name',
'description',
'type',
'payload',
)
def __call__(self, path):
return self.parse(path)
- def _load_linter(self, path):
- # Ensure parent module is present otherwise we'll (likely) get
- # an error due to unknown parent.
- parent_module = 'mozlint.linters'
- if parent_module not in sys.modules:
- mod = imp.new_module(parent_module)
- sys.modules[parent_module] = mod
-
- write_bytecode = sys.dont_write_bytecode
- sys.dont_write_bytecode = True
-
- module_name = '{}.{}'.format(parent_module, uuid.uuid1().get_hex())
- imp.load_source(module_name, path)
-
- sys.dont_write_bytecode = write_bytecode
-
- mod = sys.modules[module_name]
-
- if not hasattr(mod, 'LINTER'):
- raise LinterParseError(path, "No LINTER definition found!")
-
- definition = mod.LINTER
- definition['path'] = path
- return definition
-
def _validate(self, linter):
missing_attrs = []
for attr in self.required_attributes:
if attr not in linter:
missing_attrs.append(attr)
if missing_attrs:
raise LinterParseError(linter['path'], "Missing required attribute(s): "
@@ -66,20 +40,30 @@ class Parser(object):
not all(isinstance(a, basestring) for a in linter[attr])):
raise LinterParseError(linter['path'], "The {} directive must be a "
"list of strings!".format(attr))
def parse(self, path):
"""Read a linter and return its LINTER definition.
:param path: Path to the linter.
- :returns: Linter definition (dict)
+ :returns: List of linter definitions ([dict])
:raises: LinterNotFound, LinterParseError
"""
if not os.path.isfile(path):
raise LinterNotFound(path)
- if not path.endswith('.lint.py'):
- raise LinterParseError(path, "Invalid filename, linters must end with '.lint.py'!")
+ if not path.endswith('.yml'):
+ raise LinterParseError(path, "Invalid filename, linters must end with '.yml'!")
+
+ with open(path) as fh:
+ config = yaml.load(fh)
- linter = self._load_linter(path)
- self._validate(linter)
- return linter
+ if not config:
+ raise LinterParseError(path, "No lint definitions found!")
+
+ linters = []
+ for name, linter in config.iteritems():
+ linter['name'] = name
+ linter['path'] = path
+ self._validate(linter)
+ linters.append(linter)
+ return linters
--- a/python/mozlint/mozlint/pathutils.py
+++ b/python/mozlint/mozlint/pathutils.py
@@ -1,13 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-from __future__ import unicode_literals
+from __future__ import unicode_literals, absolute_import
import os
from mozpack import path as mozpath
from mozpack.files import FileFinder
class FilterPath(object):
@@ -149,8 +149,30 @@ def filterpaths(paths, linter, **lintarg
path.exclude = [e.path for e in excludeglobs]
for pattern in includeglobs:
for p, f in path.finder.find(pattern.path):
keep.add(path.join(p))
# Only pass paths we couldn't exclude here to the underlying linter
lintargs['exclude'] = [f.path for f in discard]
return [f.path for f in keep]
+
+
+def findobject(path):
+ """
+ Find a Python object given a path of the form <modulepath>:<objectpath>.
+ Conceptually equivalent to
+
+ def find_object(modulepath, objectpath):
+ import <modulepath> as mod
+ return mod.<objectpath>
+ """
+ if path.count(':') != 1:
+ raise ValueError(
+ 'python path {!r} does not have the form "module:object"'.format(path))
+
+ modulepath, objectpath = path.split(':')
+ obj = __import__(modulepath)
+ for a in modulepath.split('.')[1:]:
+ obj = getattr(obj, a)
+ for a in objectpath.split('.'):
+ obj = getattr(obj, a)
+ return obj
--- a/python/mozlint/mozlint/result.py
+++ b/python/mozlint/mozlint/result.py
@@ -60,29 +60,29 @@ class ResultEncoder(JSONEncoder):
json.dumps(results, cls=ResultEncoder)
"""
def default(self, o):
if isinstance(o, ResultContainer):
return {a: getattr(o, a) for a in o.__slots__}
return JSONEncoder.default(self, o)
-def from_linter(lintobj, **kwargs):
- """Create a :class:`~result.ResultContainer` from a LINTER definition.
+def from_config(config, **kwargs):
+ """Create a :class:`~result.ResultContainer` from a linter config.
- Convenience method that pulls defaults from a LINTER
- definition and forwards them.
+ Convenience method that pulls defaults from a linter
+ config and forwards them.
- :param lintobj: LINTER obj as defined in a .lint.py file
+ :param config: linter config as defined in a .yml file
:param kwargs: same as :class:`~result.ResultContainer`
:returns: :class:`~result.ResultContainer` object
"""
attrs = {}
for attr in ResultContainer.__slots__:
- attrs[attr] = kwargs.get(attr, lintobj.get(attr))
+ attrs[attr] = kwargs.get(attr, config.get(attr))
if not attrs['linter']:
- attrs['linter'] = lintobj.get('name')
+ attrs['linter'] = config.get('name')
if not attrs['message']:
- attrs['message'] = lintobj.get('description')
+ attrs['message'] = config.get('description')
return ResultContainer(**attrs)
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -14,42 +14,36 @@ from Queue import Empty
from .errors import LintersNotConfigured
from .parser import Parser
from .types import supported_types
from .vcs import VCSFiles
def _run_linters(queue, paths, **lintargs):
- parse = Parser()
results = defaultdict(list)
failed = []
while True:
try:
# The astute reader may wonder what is preventing the worker from
- # grabbing the next linter from the queue after a SIGINT. Because
- # this is a Manager.Queue(), it is itself in a child process which
- # also received SIGINT. By the time the worker gets back here, the
- # Queue is dead and IOError is raised.
- linter_path = queue.get(False)
+ # grabbing the next linter config from the queue after a SIGINT.
+ # Because this is a Manager.Queue(), it is itself in a child process
+ # which also received SIGINT. By the time the worker gets back here,
+ # the Queue is dead and IOError is raised.
+ config = queue.get(False)
except (Empty, IOError):
return results, failed
- # Ideally we would pass the entire LINTER definition as an argument
- # to the worker instead of re-parsing it. But passing a function from
- # a dynamically created module (with imp) does not seem to be possible
- # with multiprocessing on Windows.
- linter = parse(linter_path)
- func = supported_types[linter['type']]
- res = func(paths, linter, **lintargs) or []
+ func = supported_types[config['type']]
+ res = func(paths, config, **lintargs) or []
if not isinstance(res, (list, tuple)):
if res:
- failed.append(linter['name'])
+ failed.append(config['name'])
continue
for r in res:
results[r.path].append(r)
def _run_worker(*args, **lintargs):
try:
@@ -87,17 +81,17 @@ class LintRoller(object):
"""Parse one or more linters and add them to the registry.
:param paths: A path or iterable of paths to linter definitions.
"""
if isinstance(paths, basestring):
paths = (paths,)
for path in paths:
- self.linters.append(self.parse(path))
+ self.linters.extend(self.parse(path))
def roll(self, paths=None, rev=None, outgoing=None, workdir=None, num_procs=None):
"""Run all of the registered linters against the specified file paths.
:param paths: An iterable of files and/or directories to lint.
:param rev: Lint all files touched by the specified revision.
:param outgoing: Lint files touched by commits that are not on the remote repository.
:param workdir: Lint all files touched in the working directory.
@@ -122,18 +116,18 @@ class LintRoller(object):
paths = paths or ['.']
paths = map(os.path.abspath, paths)
# Set up multiprocessing
m = Manager()
queue = m.Queue()
- for linter in self.linters:
- queue.put(linter['path'])
+ for config in self.linters:
+ queue.put(config)
num_procs = num_procs or cpu_count()
num_procs = min(num_procs, len(self.linters))
pool = Pool(num_procs)
all_results = defaultdict(list)
workers = []
for i in range(num_procs):
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -7,44 +7,44 @@ from __future__ import unicode_literals
import re
import sys
from abc import ABCMeta, abstractmethod
from mozlog import get_default_logger, commandline, structuredlog
from mozlog.reader import LogHandler
from . import result
-from .pathutils import filterpaths
+from .pathutils import filterpaths, findobject
class BaseType(object):
"""Abstract base class for all types of linters."""
__metaclass__ = ABCMeta
batch = False
- def __call__(self, paths, linter, **lintargs):
- """Run `linter` against `paths` with `lintargs`.
+ def __call__(self, paths, config, **lintargs):
+ """Run linter defined by `config` against `paths` with `lintargs`.
:param paths: Paths to lint. Can be a file or directory.
- :param linter: Linter definition paths are being linted against.
+ :param config: Linter config the paths are being linted against.
:param lintargs: External arguments to the linter not defined in
the definition, but passed in by a consumer.
:returns: A list of :class:`~result.ResultContainer` objects.
"""
- paths = filterpaths(paths, linter, **lintargs)
+ paths = filterpaths(paths, config, **lintargs)
if not paths:
return
if self.batch:
- return self._lint(paths, linter, **lintargs)
+ return self._lint(paths, config, **lintargs)
errors = []
try:
for p in paths:
- result = self._lint(p, linter, **lintargs)
+ result = self._lint(p, config, **lintargs)
if result:
errors.extend(result)
except KeyboardInterrupt:
pass
return errors
@abstractmethod
def _lint(self, path):
@@ -58,26 +58,26 @@ class LineType(BaseType):
payload against each line one by one.
"""
__metaclass__ = ABCMeta
@abstractmethod
def condition(payload, line):
pass
- def _lint(self, path, linter, **lintargs):
- payload = linter['payload']
+ def _lint(self, path, config, **lintargs):
+ payload = config['payload']
with open(path, 'r') as fh:
lines = fh.readlines()
errors = []
for i, line in enumerate(lines):
if self.condition(payload, line):
- errors.append(result.from_linter(linter, path=path, lineno=i+1))
+ errors.append(result.from_config(config, path=path, lineno=i+1))
return errors
class StringType(LineType):
"""Linter type that checks whether a substring is found."""
def condition(self, payload, line):
@@ -94,48 +94,50 @@ class RegexType(LineType):
class ExternalType(BaseType):
"""Linter type that runs an external function.
The function is responsible for properly formatting the results
into a list of :class:`~result.ResultContainer` objects.
"""
batch = True
- def _lint(self, files, linter, **lintargs):
- payload = linter['payload']
- return payload(files, **lintargs)
+ def _lint(self, files, config, **lintargs):
+ func = findobject(config['payload'])
+ return func(files, config, **lintargs)
class LintHandler(LogHandler):
- def __init__(self, linter):
- self.linter = linter
+ def __init__(self, config):
+ self.config = config
self.results = []
def lint(self, data):
- self.results.append(result.from_linter(self.linter, **data))
+ self.results.append(result.from_config(self.config, **data))
class StructuredLogType(BaseType):
batch = True
- def _lint(self, files, linter, **lintargs):
- payload = linter["payload"]
- handler = LintHandler(linter)
- logger = linter.get("logger")
+ def _lint(self, files, config, **lintargs):
+ handler = LintHandler(config)
+ logger = config.get("logger")
if logger is None:
logger = get_default_logger()
if logger is None:
- logger = structuredlog.StructuredLogger(linter["name"])
+ logger = structuredlog.StructuredLogger(config["name"])
commandline.setup_logging(logger, {}, {"mach": sys.stdout})
logger.add_handler(handler)
+
+ func = findobject(config["payload"])
try:
- payload(files, logger, **lintargs)
+ func(files, config, logger, **lintargs)
except KeyboardInterrupt:
pass
return handler.results
+
supported_types = {
'string': StringType(),
'regex': RegexType(),
'external': ExternalType(),
'structured_log': StructuredLogType()
}
"""Mapping of type string to an associated instance."""
--- a/python/mozlint/test/conftest.py
+++ b/python/mozlint/test/conftest.py
@@ -1,13 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
+import sys
import pytest
from mozlint import LintRoller
here = os.path.abspath(os.path.dirname(__file__))
@@ -27,16 +28,18 @@ def filedir():
def files(filedir, request):
suffix_filter = getattr(request.module, 'files', [''])
return [os.path.join(filedir, p) for p in os.listdir(filedir)
if any(p.endswith(suffix) for suffix in suffix_filter)]
@pytest.fixture(scope='session')
def lintdir():
- return os.path.join(here, 'linters')
+ lintdir = os.path.join(here, 'linters')
+ sys.path.insert(0, lintdir)
+ return lintdir
@pytest.fixture(scope='module')
def linters(lintdir, request):
suffix_filter = getattr(request.module, 'linters', ['.lint.py'])
return [os.path.join(lintdir, p) for p in os.listdir(lintdir)
if any(p.endswith(suffix) for suffix in suffix_filter)]
rename from python/mozlint/test/linters/badreturncode.lint.py
rename to python/mozlint/test/linters/badreturncode.yml
--- a/python/mozlint/test/linters/badreturncode.lint.py
+++ b/python/mozlint/test/linters/badreturncode.yml
@@ -1,21 +1,7 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-def lint(files, **lintargs):
- return 1
-
-
-LINTER = {
- 'name': "BadReturnCodeLinter",
- 'description': "Returns an error code no matter what",
- 'include': [
- 'files',
- ],
- 'type': 'external',
- 'extensions': ['.js', '.jsm'],
- 'payload': lint,
-}
+BadReturnCodeLinter:
+ description: Returns an error code no matter what
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:badreturncode
rename from python/mozlint/test/linters/explicit_path.lint.py
rename to python/mozlint/test/linters/explicit_path.yml
--- a/python/mozlint/test/linters/explicit_path.lint.py
+++ b/python/mozlint/test/linters/explicit_path.yml
@@ -1,13 +1,7 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "ExplicitPathLinter",
- 'description': "Only lint a specific file name",
- 'rule': 'no-foobar',
- 'include': [
- 'no_foobar.js',
- ],
- 'type': 'string',
- 'payload': 'foobar',
-}
+ExplicitPathLinter:
+ description: Only lint a specific file name
+ rule: no-foobar
+ include:
+ - no_foobar.js
+ type: string
+ payload: foobar
new file mode 100644
--- /dev/null
+++ b/python/mozlint/test/linters/external.py
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozlint import result
+from mozlint.errors import LintException
+
+
+def badreturncode(files, config, **lintargs):
+ return 1
+
+
+def external(files, config, **lintargs):
+ results = []
+ for path in files:
+ with open(path, 'r') as fh:
+ for i, line in enumerate(fh.readlines()):
+ if 'foobar' in line:
+ results.append(result.from_config(
+ config, path=path, lineno=i+1, column=1, rule="no-foobar"))
+ return results
+
+
+def raises(files, config, **lintargs):
+ raise LintException("Oh no something bad happened!")
+
+
+def structured(files, config, logger, **kwargs):
+ for path in files:
+ with open(path, 'r') as fh:
+ for i, line in enumerate(fh.readlines()):
+ if 'foobar' in line:
+ logger.lint_error(path=path,
+ lineno=i+1,
+ column=1,
+ rule="no-foobar")
rename from python/mozlint/test/linters/external.lint.py
rename to python/mozlint/test/linters/external.yml
--- a/python/mozlint/test/linters/external.lint.py
+++ b/python/mozlint/test/linters/external.yml
@@ -1,30 +1,7 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from mozlint import result
-
-
-def lint(files, **lintargs):
- results = []
- for path in files:
- with open(path, 'r') as fh:
- for i, line in enumerate(fh.readlines()):
- if 'foobar' in line:
- results.append(result.from_linter(
- LINTER, path=path, lineno=i+1, column=1, rule="no-foobar"))
- return results
-
-
-LINTER = {
- 'name': "ExternalLinter",
- 'description': "It's bad to have the string foobar in js files.",
- 'include': [
- 'files',
- ],
- 'type': 'external',
- 'extensions': ['.js', '.jsm'],
- 'payload': lint,
-}
+ExternalLinter:
+ description: It's bad to have the string foobar in js files.
+ include:
+ - files
+ type: external
+ extensions: ['.js', '.jsm']
+ payload: external:external
rename from python/mozlint/test/linters/invalid_exclude.lint.py
rename to python/mozlint/test/linters/invalid_exclude.yml
--- a/python/mozlint/test/linters/invalid_exclude.lint.py
+++ b/python/mozlint/test/linters/invalid_exclude.yml
@@ -1,10 +1,5 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "BadExcludeLinter",
- 'description': "Has an invalid exclude directive.",
- 'exclude': [0, 1], # should be a list of strings
- 'type': 'string',
- 'payload': 'foobar',
-}
+BadExcludeLinter:
+ description: Has an invalid exclude directive.
+ exclude: [0, 1] # should be a list of strings
+ type: string
+ payload: foobar
rename from python/mozlint/test/linters/invalid_extension.lnt
rename to python/mozlint/test/linters/invalid_extension.ym
--- a/python/mozlint/test/linters/invalid_extension.lnt
+++ b/python/mozlint/test/linters/invalid_extension.ym
@@ -1,9 +1,4 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "BadExtensionLinter",
- 'description': "Has an invalid file extension.",
- 'type': 'string',
- 'payload': 'foobar',
-}
+BadExtensionLinter:
+ description: Has an invalid file extension.
+ type: string
+ payload: foobar
rename from python/mozlint/test/linters/invalid_include.lint.py
rename to python/mozlint/test/linters/invalid_include.yml
--- a/python/mozlint/test/linters/invalid_include.lint.py
+++ b/python/mozlint/test/linters/invalid_include.yml
@@ -1,10 +1,5 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "BadIncludeLinter",
- 'description': "Has an invalid include directive.",
- 'include': 'should be a list',
- 'type': 'string',
- 'payload': 'foobar',
-}
+BadIncludeLinter:
+ description: Has an invalid include directive.
+ include: should be a list
+ type: string
+ payload: foobar
rename from python/mozlint/test/linters/invalid_type.lint.py
rename to python/mozlint/test/linters/invalid_type.yml
--- a/python/mozlint/test/linters/invalid_type.lint.py
+++ b/python/mozlint/test/linters/invalid_type.yml
@@ -1,9 +1,4 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "BadTypeLinter",
- 'description': "Has an invalid type.",
- 'type': 'invalid',
- 'payload': 'foobar',
-}
+BadTypeLinter:
+ description: Has an invalid type.
+ type: invalid
+ payload: foobar
rename from python/mozlint/test/linters/missing_attrs.lint.py
rename to python/mozlint/test/linters/missing_attrs.yml
--- a/python/mozlint/test/linters/missing_attrs.lint.py
+++ b/python/mozlint/test/linters/missing_attrs.yml
@@ -1,7 +1,2 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "MissingAttrsLinter",
- 'description': "Missing type and payload",
-}
+MissingAttrsLinter:
+ description: Missing type and payload
rename from python/mozlint/test/linters/missing_definition.lint.py
rename to python/mozlint/test/linters/missing_definition.yml
--- a/python/mozlint/test/linters/missing_definition.lint.py
+++ b/python/mozlint/test/linters/missing_definition.yml
@@ -1,4 +1,1 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-# No LINTER variable
+# No definition
rename from python/mozlint/test/linters/raises.lint.py
rename to python/mozlint/test/linters/raises.yml
--- a/python/mozlint/test/linters/raises.lint.py
+++ b/python/mozlint/test/linters/raises.yml
@@ -1,19 +1,4 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from mozlint.errors import LintException
-
-
-def lint(files, **lintargs):
- raise LintException("Oh no something bad happened!")
-
-
-LINTER = {
- 'name': "RaisesLinter",
- 'description': "Raises an exception",
- 'type': 'external',
- 'payload': lint,
-}
+RaisesLinter:
+ description: Raises an exception
+ type: external
+ payload: external:raises
rename from python/mozlint/test/linters/regex.lint.py
rename to python/mozlint/test/linters/regex.yml
--- a/python/mozlint/test/linters/regex.lint.py
+++ b/python/mozlint/test/linters/regex.yml
@@ -1,15 +1,8 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "RegexLinter",
- 'description': "Make sure the string 'foobar' never appears "
- "in a js variable file because it is bad.",
- 'rule': 'no-foobar',
- 'include': [
- '**/*.js',
- '**/*.jsm',
- ],
- 'type': 'regex',
- 'payload': 'foobar',
-}
+RegexLinter:
+ description: Make sure the string foobar never appears in a js variable file because it is bad.
+ rule: no-foobar
+ include:
+ - '**/*.js'
+ - '**/*.jsm'
+ type: regex
+ payload: foobar
rename from python/mozlint/test/linters/string.lint.py
rename to python/mozlint/test/linters/string.yml
--- a/python/mozlint/test/linters/string.lint.py
+++ b/python/mozlint/test/linters/string.yml
@@ -1,15 +1,8 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-
-LINTER = {
- 'name': "StringLinter",
- 'description': "Make sure the string 'foobar' never appears "
- "in browser js files because it is bad.",
- 'rule': 'no-foobar',
- 'include': [
- '**/*.js',
- '**/*.jsm',
- ],
- 'type': 'string',
- 'payload': 'foobar',
-}
+StringLinter:
+ description: Make sure the string foobar never appears in browser js files because it is bad
+ rule: no-foobar
+ include:
+ - '**/*.js'
+ - '**/*.jsm'
+ type: string
+ payload: foobar
rename from python/mozlint/test/linters/structured.lint.py
rename to python/mozlint/test/linters/structured.yml
--- a/python/mozlint/test/linters/structured.lint.py
+++ b/python/mozlint/test/linters/structured.yml
@@ -1,28 +1,7 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-def lint(files, logger, **kwargs):
- for path in files:
- with open(path, 'r') as fh:
- for i, line in enumerate(fh.readlines()):
- if 'foobar' in line:
- logger.lint_error(path=path,
- lineno=i+1,
- column=1,
- rule="no-foobar")
-
-
-LINTER = {
- 'name': "StructuredLinter",
- 'description': "It's bad to have the string foobar in js files.",
- 'include': [
- 'files',
- ],
- 'type': 'structured_log',
- 'extensions': ['.js', '.jsm'],
- 'payload': lint,
-}
+StructuredLinter:
+ description: "It's bad to have the string foobar in js files."
+ include:
+ - files
+ type: structured_log
+ extensions: ['.js', '.jsm']
+ payload: external:structured
--- a/python/mozlint/test/test_parser.py
+++ b/python/mozlint/test/test_parser.py
@@ -20,31 +20,35 @@ def parse(lintdir):
def _parse(name):
path = os.path.join(lintdir, name)
return parser(path)
return _parse
def test_parse_valid_linter(parse):
- lintobj = parse('string.lint.py')
+ lintobj = parse('string.yml')
+ assert isinstance(lintobj, list)
+ assert len(lintobj) == 1
+
+ lintobj = lintobj[0]
assert isinstance(lintobj, dict)
assert 'name' in lintobj
assert 'description' in lintobj
assert 'type' in lintobj
assert 'payload' in lintobj
@pytest.mark.parametrize('linter', [
- 'invalid_type.lint.py',
- 'invalid_extension.lnt',
- 'invalid_include.lint.py',
- 'invalid_exclude.lint.py',
- 'missing_attrs.lint.py',
- 'missing_definition.lint.py',
+ 'invalid_type.yml',
+ 'invalid_extension.ym',
+ 'invalid_include.yml',
+ 'invalid_exclude.yml',
+ 'missing_attrs.yml',
+ 'missing_definition.yml',
])
def test_parse_invalid_linter(parse, linter):
with pytest.raises(LinterParseError):
parse(linter)
def test_parse_non_existent_linter(parse):
with pytest.raises(LinterNotFound):
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -9,17 +9,17 @@ import pytest
from mozlint import ResultContainer
from mozlint.errors import LintersNotConfigured, LintException
here = os.path.abspath(os.path.dirname(__file__))
-linters = ('string.lint.py', 'regex.lint.py', 'external.lint.py')
+linters = ('string.yml', 'regex.yml', 'external.yml')
def test_roll_no_linters_configured(lint, files):
with pytest.raises(LintersNotConfigured):
lint.roll(files)
def test_roll_successful(lint, linters, files):
@@ -37,17 +37,17 @@ def test_roll_successful(lint, linters,
assert len(errors) == 6
container = errors[0]
assert isinstance(container, ResultContainer)
assert container.rule == 'no-foobar'
def test_roll_catch_exception(lint, lintdir, files):
- lint.read(os.path.join(lintdir, 'raises.lint.py'))
+ lint.read(os.path.join(lintdir, 'raises.yml'))
# suppress printed traceback from test output
old_stderr = sys.stderr
sys.stderr = open(os.devnull, 'w')
with pytest.raises(LintException):
lint.roll(files)
sys.stderr = old_stderr
@@ -58,24 +58,24 @@ def test_roll_with_excluded_path(lint, l
lint.read(linters)
result = lint.roll(files)
assert len(result) == 0
assert lint.failed == []
def test_roll_with_invalid_extension(lint, lintdir, filedir):
- lint.read(os.path.join(lintdir, 'external.lint.py'))
+ lint.read(os.path.join(lintdir, 'external.yml'))
result = lint.roll(os.path.join(filedir, 'foobar.py'))
assert len(result) == 0
assert lint.failed == []
def test_roll_with_failure_code(lint, lintdir, files):
- lint.read(os.path.join(lintdir, 'badreturncode.lint.py'))
+ lint.read(os.path.join(lintdir, 'badreturncode.yml'))
assert lint.failed is None
result = lint.roll(files)
assert len(result) == 0
assert lint.failed == ['BadReturnCodeLinter']
if __name__ == '__main__':
--- a/python/mozlint/test/test_types.py
+++ b/python/mozlint/test/test_types.py
@@ -13,20 +13,20 @@ from mozlint.result import ResultContain
@pytest.fixture
def path(filedir):
def _path(name):
return os.path.join(filedir, name)
return _path
@pytest.fixture(params=[
- 'string.lint.py',
- 'regex.lint.py',
- 'external.lint.py',
- 'structured.lint.py'])
+ 'string.yml',
+ 'regex.yml',
+ 'external.yml',
+ 'structured.yml'])
def linter(lintdir, request):
return os.path.join(lintdir, request.param)
def test_linter_types(lint, linter, files, path):
lint.read(linter)
result = lint.roll(files)
assert isinstance(result, dict)
@@ -36,17 +36,17 @@ def test_linter_types(lint, linter, file
result = result[path('foobar.js')][0]
assert isinstance(result, ResultContainer)
name = os.path.basename(linter).split('.')[0]
assert result.linter.lower().startswith(name)
def test_no_filter(lint, lintdir, files):
- lint.read(os.path.join(lintdir, 'explicit_path.lint.py'))
+ lint.read(os.path.join(lintdir, 'explicit_path.yml'))
result = lint.roll(files)
assert len(result) == 0
lint.lintargs['use_filters'] = False
result = lint.roll(files)
assert len(result) == 2
--- a/tools/lint/docs/create.rst
+++ b/tools/lint/docs/create.rst
@@ -1,35 +1,34 @@
Adding a New Linter to the Tree
===============================
-A linter is a python file with a ``.lint`` extension and a global dict called LINTER. Depending on how
-complex it is, there may or may not be any actual python code alongside the LINTER definition.
+A linter is a yaml file with a ``.yml`` extension. Depending on how the type of linter, there may
+be python code alongside the definition, pointed to by the 'payload' attribute.
Here's a trivial example:
-no-eval.lint
+no-eval.yml
-.. code-block:: python
+.. code-block::
- LINTER = {
- 'name': 'EvalLinter',
- 'description': "Ensures the string 'eval' doesn't show up."
- 'include': "**/*.js",
- 'type': 'string',
- 'payload': 'eval',
- }
+ EvalLinter:
+ description: Ensures the string eval doesn't show up.
+ include:
+ - "**/*.js"
+ type: string
+ payload: eval
-Now ``no-eval.lint`` gets passed into :func:`LintRoller.read`.
+Now ``no-eval.yml`` gets passed into :func:`LintRoller.read`.
Linter Types
------------
-There are three types of linters, though more may be added in the future.
+There are four types of linters, though more may be added in the future.
1. string - fails if substring is found
2. regex - fails if regex matches
3. external - fails if a python function returns a non-empty result list
4. structured_log - fails if a mozlog logger emits any lint_error or lint_warning log messages
As seen from the example above, string and regex linters are very easy to create, but they
should be avoided if possible. It is much better to use a context aware linter for the language you
@@ -37,75 +36,77 @@ are trying to lint. For example, use esl
files, etc.
Which brings us to the third and most interesting type of linter,
external. External linters call an arbitrary python function which is
responsible for not only running the linter, but ensuring the results
are structured properly. For example, an external type could shell out
to a 3rd party linter, collect the output and format it into a list of
:class:`ResultContainer` objects. The signature for this python
-function is ``lint(files, **kwargs)``, where ``files`` is a list of
-files to lint.
+function is ``lint(files, config, **kwargs)``, where ``files`` is a list of
+files to lint and ``config`` is the linter definition defined in the ``.yml``
+file.
Structured log linters are much like external linters, but suitable
for cases where the linter code is using mozlog and emits
``lint_error`` or ``lint_warning`` logging messages when the lint
fails. This is recommended for writing novel gecko-specific lints. In
-this case the signature for lint functions is ``lint(files, logger,
+this case the signature for lint functions is ``lint(files, config, logger,
**kwargs)``.
-LINTER Definition
+
+Linter Definition
-----------------
-Each ``.lint`` file must have a variable called LINTER which is a dict containing metadata about the
-linter. Here are the supported keys:
+Each ``.yml`` file must have at least one linter defined in it. Here are the supported keys:
-* name - The name of the linter (required)
* description - A brief description of the linter's purpose (required)
* type - One of 'string', 'regex' or 'external' (required)
* payload - The actual linting logic, depends on the type (required)
* include - A list of glob patterns that must be matched (optional)
* exclude - A list of glob patterns that must not be matched (optional)
* extensions - A list of file extensions to be considered (optional)
* setup - A function that sets up external dependencies (optional)
-In addition to the above, some ``.lint`` files correspond to a single lint rule. For these, the
+In addition to the above, some ``.yml`` files correspond to a single lint rule. For these, the
following additional keys may be specified:
* message - A string to print on infraction (optional)
* hint - A string with a clue on how to fix the infraction (optional)
* rule - An id string for the lint rule (optional)
* level - The severity of the infraction, either 'error' or 'warning' (optional)
For structured_log lints the following additional keys apply:
* logger - A StructuredLog object to use for logging. If not supplied
one will be created (optional)
+
Example
-------
-Here is an example of an external linter that shells out to the python flake8 linter:
+Here is an example of an external linter that shells out to the python flake8 linter,
+let's call the file ``flake8_lint.py``:
.. code-block:: python
import json
import os
import subprocess
from collections import defaultdict
from mozlint import result
FLAKE8_NOT_FOUND = """
Could not find flake8! Install flake8 and try again.
""".strip()
- def lint(files, **lintargs):
+ def lint(files, config, **lintargs):
import which
binary = os.environ.get('FLAKE8')
if not binary:
try:
binary = which.which('flake8')
except which.WhichError:
print(FLAKE8_NOT_FOUND)
@@ -134,20 +135,27 @@ Here is an example of an external linter
# parse level out of the id string
if 'code' in res and res['code'].startswith('W'):
res['level'] = 'warning'
# result.from_linter is a convenience method that
# creates a ResultContainer using a LINTER definition
# to populate some defaults.
- results.append(result.from_linter(LINTER, **res))
+ results.append(result.from_config(config, **res))
return results
+Now here is the linter definition that would call it:
- LINTER = {
- 'name': "flake8",
- 'description': "Python linter",
- 'include': ['**/*.py'],
- 'type': 'external',
- 'payload': lint,
- }
+.. code-block::
+
+ flake8:
+ description: Python linter
+ include:
+ - '**/*.py'
+ type: external
+ payload: flake8_lint:lint
+
+Notice the payload has two parts, delimited by ':'. The first is the module path, which
+``mozlint`` will attempt to import (e.g, the name of a function to call). The second is
+the object path within that module. It is up to consumers of ``mozlint`` to ensure the
+module is in ``sys.path``. Structured log linters use the same import mechanism.
--- a/tools/lint/docs/index.rst
+++ b/tools/lint/docs/index.rst
@@ -5,17 +5,17 @@ Linters are used in mozilla-central to h
wide variety of languages in use and the varying style preferences per team, this is not an easy
task. In addition, linters should be runnable from editors, from the command line, from review tools
and from continuous integration. It's easy to see how the complexity of running all of these
different kinds of linters in all of these different places could quickly balloon out of control.
``Mozlint`` is a library that accomplishes two goals:
1. It provides a standard method for adding new linters to the tree, which can be as easy as
- defining a json object in a ``.lint`` file. This helps keep lint related code localized, and
+ defining a config object in a ``.yml`` file. This helps keep lint related code localized, and
prevents different teams from coming up with their own unique lint implementations.
2. It provides a streamlined interface for running all linters at once. Instead of running N
different lint commands to test your patch, a single ``mach lint`` command will automatically run
all applicable linters. This means there is a single API surface that other tools can use to
invoke linters.
``Mozlint`` isn't designed to be used directly by end users. Instead, it can be consumed by things
like mach, mozreview and taskcluster.