Bug 1289805 - Resolve vcs arguments directly in LintRoller class, r?smacleod
Previously, vcs related stuff was resolved in the cli.py module. But it's possible
for consumers to bypass the cli and instantiate a LintRoller directly. In fact this
is what the mozlint tests do.
Now that we always try to find the vcs root, calling into vcs is no longer optional.
This patch moves the VCSFiles class to a new vcs.py module and makes LintRoller
responsible for instantiating it instead of cli.py.
MozReview-Commit-ID: 5yA3gDZ1UGM
--- a/python/mozlint/mozlint/cli.py
+++ b/python/mozlint/mozlint/cli.py
@@ -1,16 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import os
-import subprocess
import sys
from argparse import ArgumentParser
SEARCH_PATHS = []
class MozlintParser(ArgumentParser):
@@ -57,72 +56,16 @@ class MozlintParser(ArgumentParser):
def __init__(self, **kwargs):
ArgumentParser.__init__(self, usage=self.__doc__, **kwargs)
for cli, args in self.arguments:
self.add_argument(*cli, **args)
-class VCSFiles(object):
- def __init__(self):
- self._root = None
- self._vcs = None
-
- @property
- def root(self):
- if self._root:
- return self._root
-
- # First check if we're in an hg repo, if not try git
- commands = (
- ['hg', 'root'],
- ['git', 'rev-parse', '--show-toplevel'],
- )
-
- for cmd in commands:
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- output = proc.communicate()[0].strip()
-
- if proc.returncode == 0:
- self._vcs = cmd[0]
- self._root = output
- return self._root
-
- @property
- def vcs(self):
- return self._vcs or (self.root and self._vcs)
-
- @property
- def is_hg(self):
- return self.vcs == 'hg'
-
- @property
- def is_git(self):
- return self.vcs == 'git'
-
- def _run(self, cmd):
- files = subprocess.check_output(cmd).split()
- return [os.path.join(self.root, f) for f in files]
-
- def by_rev(self, rev):
- if self.is_hg:
- return self._run(['hg', 'log', '-T', '{files % "\\n{file}"}', '-r', rev])
- elif self.is_git:
- return self._run(['git', 'diff', '--name-only', rev])
- return []
-
- def by_workdir(self):
- if self.is_hg:
- return self._run(['hg', 'status', '-amn'])
- elif self.is_git:
- return self._run(['git', 'diff', '--name-only'])
- return []
-
-
def find_linters(linters=None):
lints = []
for search_path in SEARCH_PATHS:
if not os.path.isdir(search_path):
continue
files = os.listdir(search_path)
for f in files:
@@ -135,29 +78,21 @@ def find_linters(linters=None):
lints.append(os.path.join(search_path, f))
return lints
def run(paths, linters, fmt, rev, workdir, **lintargs):
from mozlint import LintRoller, formatters
- # Calculate files from VCS
- vcs = VCSFiles()
- if rev:
- paths.extend(vcs.by_rev(rev))
- if workdir:
- paths.extend(vcs.by_workdir())
- paths = paths or ['.']
-
lint = LintRoller(**lintargs)
lint.read(find_linters(linters))
# run all linters
- results = lint.roll(paths)
+ results = lint.roll(paths, rev=rev, workdir=workdir)
formatter = formatters.get(fmt)
print(formatter(results))
return 1 if results else 0
if __name__ == '__main__':
parser = MozlintParser()
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -1,27 +1,29 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
+import os
import signal
import traceback
from collections import defaultdict
from Queue import Empty
from multiprocessing import (
Manager,
Pool,
cpu_count,
)
from .errors import LintersNotConfigured
from .types import supported_types
from .parser import Parser
+from .vcs import VCSFiles
def _run_linters(queue, paths, **lintargs):
parse = Parser()
results = defaultdict(list)
while True:
try:
@@ -62,44 +64,58 @@ def _run_worker(*args, **lintargs):
class LintRoller(object):
"""Registers and runs linters.
:param lintargs: Arguments to pass to the underlying linter(s).
"""
def __init__(self, **lintargs):
self.parse = Parser()
+ self.vcs = VCSFiles()
+
self.linters = []
self.lintargs = lintargs
+ self.lintargs['root'] = self.vcs.root or os.getcwd()
def read(self, paths):
"""Parse one or more linters and add them to the registry.
:param paths: A path or iterable of paths to linter definitions.
"""
if isinstance(paths, basestring):
paths = (paths,)
for path in paths:
self.linters.append(self.parse(path))
- def roll(self, paths, num_procs=None):
+ def roll(self, paths=None, rev=None, workdir=None, num_procs=None):
"""Run all of the registered linters against the specified file paths.
:param paths: An iterable of files and/or directories to lint.
+ :param rev: Lint all files touched by the specified revision.
+ :param workdir: Lint all files touched in the working directory.
:param num_procs: The number of processes to use. Default: cpu count
:return: A dictionary with file names as the key, and a list of
:class:`~result.ResultContainer`s as the value.
"""
+ paths = paths or []
+ if isinstance(paths, basestring):
+ paths = [paths]
+
if not self.linters:
raise LintersNotConfigured
- if isinstance(paths, basestring):
- paths = [paths]
+ # Calculate files from VCS
+ if rev:
+ paths.extend(self.vcs.by_rev(rev))
+ if workdir:
+ paths.extend(self.vcs.by_workdir())
+ paths = paths or ['.']
+ # Set up multiprocessing
m = Manager()
queue = m.Queue()
for linter in self.linters:
queue.put(linter['path'])
num_procs = num_procs or cpu_count()
num_procs = min(num_procs, len(self.linters))
new file mode 100644
--- /dev/null
+++ b/python/mozlint/mozlint/vcs.py
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+
+
+class VCSFiles(object):
+ def __init__(self):
+ self._root = None
+ self._vcs = None
+
+ @property
+ def root(self):
+ if self._root:
+ return self._root
+
+ # First check if we're in an hg repo, if not try git
+ commands = (
+ ['hg', 'root'],
+ ['git', 'rev-parse', '--show-toplevel'],
+ )
+
+ for cmd in commands:
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ output = proc.communicate()[0].strip()
+
+ if proc.returncode == 0:
+ self._vcs = cmd[0]
+ self._root = output
+ return self._root
+
+ @property
+ def vcs(self):
+ return self._vcs or (self.root and self._vcs)
+
+ @property
+ def is_hg(self):
+ return self.vcs == 'hg'
+
+ @property
+ def is_git(self):
+ return self.vcs == 'git'
+
+ def _run(self, cmd):
+ files = subprocess.check_output(cmd).split()
+ return [os.path.join(self.root, f) for f in files]
+
+ def by_rev(self, rev):
+ if self.is_hg:
+ return self._run(['hg', 'log', '-T', '{files % "\\n{file}"}', '-r', rev])
+ elif self.is_git:
+ return self._run(['git', 'diff', '--name-only', rev])
+ return []
+
+ def by_workdir(self):
+ if self.is_hg:
+ return self._run(['hg', 'status', '-amn'])
+ elif self.is_git:
+ return self._run(['git', 'diff', '--name-only'])
+ return []
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -58,17 +58,17 @@ class TestLintRoller(TestCase):
# suppress printed traceback from test output
old_stderr = sys.stderr
sys.stderr = open(os.devnull, 'w')
with self.assertRaises(LintException):
self.lint.roll(self.files)
sys.stderr = old_stderr
def test_roll_with_excluded_path(self):
- self.lint.lintargs = {'exclude': ['**/foobar.js']}
+ self.lint.lintargs.update({'exclude': ['**/foobar.js']})
self.lint.read(self.linters)
result = self.lint.roll(self.files)
self.assertEqual(len(result), 0)
if __name__ == '__main__':