Bug 1285555 - [flake8] Automatically install/update flake8 package to correct version r?ahal
Changed flake8 installation strategy, try to update it using testings/lint/flake8_requirements.txt
moved flake8_requirements to tools/lint/flake8 directory
extracted common pip invocations to _run_pip function
minor fixes
fixed calls to check_process (concatenating list to tuple)
removed update_pip function and its error message variable
if flake8 installation reports and error print pip command output - replaced out with e.output in print function
MozReview-Commit-ID: Fmq22aL8dMq
--- a/testing/docker/lint/Dockerfile
+++ b/testing/docker/lint/Dockerfile
@@ -9,16 +9,18 @@ RUN mkdir /build
ADD topsrcdir/testing/docker/recipes/tooltool.py /build/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
# %include testing/docker/recipes/install-mercurial.sh
ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
ADD system-setup.sh /tmp/system-setup.sh
+# %include tools/lint/flake8/flake8_requirements.txt
+ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt
RUN bash /tmp/system-setup.sh
# %include testing/docker/recipes/run-task
ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/*
# Set variable normally configured at login, by the shells parent process, these
# are taken from GNU su manual
--- a/testing/docker/lint/system-setup.sh
+++ b/testing/docker/lint/system-setup.sh
@@ -56,24 +56,13 @@ tar -C /usr/local --strip-components 1 -
node -v # verify
npm -v
###
# Flake8 Setup
###
cd /setup
-cat >requirements.txt <<'EOF'
-flake8==2.5.4 \
- --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
-mccabe==0.4.0 \
- --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
-pep8==1.7.0 \
- --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
-pyflakes==1.2.3 \
- --hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da
-EOF
-pip install --require-hashes -r requirements.txt
+pip install --require-hashes -r /tmp/flake8_requirements.txt
cd /
rm -rf /setup
-
--- a/tools/lint/flake8.lint
+++ b/tools/lint/flake8.lint
@@ -2,29 +2,39 @@
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import signal
+import subprocess
import which
from mozprocess import ProcessHandler
from mozlint import result
+here = os.path.abspath(os.path.dirname(__file__))
+FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8', 'flake8_requirements.txt')
+
FLAKE8_NOT_FOUND = """
Could not find flake8! Install flake8 and try again.
- $ pip install flake8
-""".strip()
+ $ pip install -U --require-hashes -r {}
+""".strip().format(FLAKE8_REQUIREMENTS_PATH)
+
+FLAKE8_INSTALL_ERROR = """
+Unable to install correct version of flake8
+Try to install it manually with:
+ $ pip install -U --require-hashes -r {}
+""".strip().format(FLAKE8_REQUIREMENTS_PATH)
LINE_OFFSETS = {
# continuation line under-indented for hanging indent
'E121': (-1, 2),
# continuation line missing indentation or outdented
'E122': (-1, 2),
# continuation line over-indented for hanging indent
'E126': (-1, 2),
@@ -78,27 +88,64 @@ def run_process(cmdargs):
signal.signal(signal.SIGINT, orig)
try:
proc.wait()
except KeyboardInterrupt:
proc.kill()
-def lint(files, **lintargs):
+def get_flake8_binary():
+ """
+ Returns the path of the first flake8 binary available
+ if not found returns None
+ """
binary = os.environ.get('FLAKE8')
- if not binary:
- try:
- binary = which.which('flake8')
- except which.WhichError:
- pass
+ if binary:
+ return binary
+
+ try:
+ return which.which('flake8')
+ except which.WhichError:
+ return None
+
+
+def _run_pip(*args):
+ """
+ Helper function that runs pip with subprocess
+ """
+ try:
+ subprocess.check_output(['pip'] + list(args),
+ stderr=subprocess.STDOUT)
+ return True
+ except subprocess.CalledProcessError as e:
+ print(e.output)
+ return False
- if not binary:
- print(FLAKE8_NOT_FOUND)
- return []
+
+def reinstall_flake8():
+ """
+ Try to install flake8 at the target version, returns True on success
+ otherwise prints the otuput of the pip command and returns False
+ """
+ if _run_pip('install', '-U',
+ '--require-hashes', '-r',
+ FLAKE8_REQUIREMENTS_PATH):
+ return True
+
+ return False
+
+
+def lint(files, **lintargs):
+
+ if not reinstall_flake8():
+ print(FLAKE8_INSTALL_ERROR)
+ return 1
+
+ binary = get_flake8_binary()
cmdargs = [
binary,
'--format', '{"path":"%(path)s","lineno":%(row)s,'
'"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
]
# Run any paths with a .flake8 file in the directory separately so
@@ -114,17 +161,16 @@ def lint(files, **lintargs):
# XXX For some reason passing in --exclude results in flake8 not using
# the local .flake8 file. So for now only pass in --exclude if there
# is no local config.
exclude = lintargs.get('exclude')
if exclude:
cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
-
if no_config:
run_process(cmdargs+no_config)
return results
LINTER = {
'name': "flake8",
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8/flake8_requirements.txt
@@ -0,0 +1,4 @@
+flake8==2.5.4 --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
+mccabe==0.4.0 --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
+pep8==1.7.0 --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
+pyflakes==1.2.3 --hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -34,16 +34,17 @@ class MachCommands(MachCommandBase):
'lint', category='devenv',
description='Run linters.',
parser=setup_argument_parser)
def lint(self, *runargs, **lintargs):
"""Run linters."""
from mozlint import cli
lintargs['exclude'] = ['obj*']
cli.SEARCH_PATHS.append(here)
+ self._activate_virtualenv()
return cli.run(*runargs, **lintargs)
@Command('eslint', category='devenv',
description='Run eslint or help configure eslint for optimal development.')
@CommandArgument('paths', default=None, nargs='*',
help="Paths to file or directories to lint, like "
"'browser/components/loop' Defaults to the "
"current directory if not given.")