Bug 1436639 - [lint] Make sure flake8 is run with same python as |mach lint| was, r?jmaher draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 07 Feb 2018 23:28:33 -0500
changeset 754421 2be49bebe3c9da11ecc546564ca88321af14be55
parent 754399 38b3c1d03a594664c6b32c35533734283c258f43
push id98852
push userahalberstadt@mozilla.com
push dateTue, 13 Feb 2018 13:28:56 +0000
reviewersjmaher
bugs1436639
milestone60.0a1
Bug 1436639 - [lint] Make sure flake8 is run with same python as |mach lint| was, r?jmaher This fixes a bug which can happen when the default version of python differs from the version of python used with mach. For example, mach explicitly looks for python2.7. This means running |mach lint -l flake8| should also run flake8 with version 2.7. But if the default is python3, and flake8 is also installed there, the subprocess call that invokes flake8 will run under python3. This can lead to errors like "undefined name 'basestring'" and other 2to3 gotchas. This patch ensures that we run the flake8 binary (and the pip for installing flake8) from the same interpreter as mach, no matter the system default. MozReview-Commit-ID: HSuMzDsAvsW
tools/lint/python/flake8.py
tools/lint/python/flake8_requirements.txt
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -1,19 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
+import platform
 import signal
 import subprocess
+import sys
 from collections import defaultdict
 
-import which
 from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 from mozlint.pathutils import get_ancestors_by_name
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
@@ -50,16 +51,23 @@ LINE_OFFSETS = {
     'E302': (-2, 3),
 }
 """Maps a flake8 error to a lineoffset tuple.
 
 The offset is of the form (lineno_offset, num_lines) and is passed
 to the lineoffset property of `ResultContainer`.
 """
 
+# We use sys.prefix to find executables as that gets modified with
+# virtualenv's activate_this.py, whereas sys.executable doesn't.
+if platform.system() == 'Windows':
+    bindir = os.path.join(sys.prefix, 'Scripts')
+else:
+    bindir = os.path.join(sys.prefix, 'bin')
+
 results = []
 
 
 class Flake8Process(ProcessHandlerMixin):
     def __init__(self, config, *args, **kwargs):
         self.config = config
         kwargs['processOutputLine'] = [self.process_line]
         ProcessHandlerMixin.__init__(self, *args, **kwargs)
@@ -85,33 +93,22 @@ class Flake8Process(ProcessHandlerMixin)
     def run(self, *args, **kwargs):
         # flake8 seems to handle SIGINT poorly. Handle it here instead
         # so we can kill the process without a cryptic traceback.
         orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
         ProcessHandlerMixin.run(self, *args, **kwargs)
         signal.signal(signal.SIGINT, orig)
 
 
-def get_flake8_binary():
-    """
-    Returns the path of the first flake8 binary available
-    if not found returns None
-    """
-    try:
-        return which.which('flake8')
-    except which.WhichError:
-        return None
-
-
 def _run_pip(*args):
     """
     Helper function that runs pip with subprocess
     """
     try:
-        subprocess.check_output(['pip'] + list(args),
+        subprocess.check_output([os.path.join(bindir, 'pip')] + list(args),
                                 stderr=subprocess.STDOUT)
         return True
     except subprocess.CalledProcessError as e:
         print(e.output)
         return False
 
 
 def reinstall_flake8():
@@ -138,19 +135,18 @@ def run_process(config, cmd):
 
 def setup(root):
     if not reinstall_flake8():
         print(FLAKE8_INSTALL_ERROR)
         return 1
 
 
 def lint(paths, config, **lintargs):
-    binary = get_flake8_binary()
     cmdargs = [
-        binary,
+        os.path.join(bindir, 'flake8'),
         '--format', '{"path":"%(path)s","lineno":%(row)s,'
                     '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
     ]
 
     # Run any paths with a .flake8 file in the directory separately so
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
--- a/tools/lint/python/flake8_requirements.txt
+++ b/tools/lint/python/flake8_requirements.txt
@@ -12,8 +12,11 @@ pycodestyle==2.3.1 \
     --hash=sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766
 enum34==1.1.6 \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850
 configparser==3.5.0 \
     --hash=sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a
+setuptools==38.5.1 \
+    --hash=sha256:7ffe771abfae419fd104f93400b61c935b5af10bfe4dfeec7a1bd495694eea35 \
+    --hash=sha256:6425484c08e99a98a42209c25c3d325f749230b55284d66192784f941a7e6628