--- a/tools/lint/py2.yml
+++ b/tools/lint/py2.yml
@@ -58,8 +58,9 @@ py2:
         - tools/power/mach_commands.py
         - tools/profiler
         - tools/rb
         - tools/update-packaging
         - xpcom
     extensions: ['py']
     type: external
     payload: python.compat:lintpy2
+    setup: python.compat:setuppy2

--- a/tools/lint/py3.yml
+++ b/tools/lint/py3.yml
@@ -43,8 +43,9 @@ py3:
         - tools/profiler
         - tools/rb
         - tools/update-packaging
         - xpcom/idl-parser
         - xpcom/typelib
     extensions: ['py']
     type: external
     payload: python.compat:lintpy3
+    setup: python.compat:setuppy3

--- a/tools/lint/python/compat.py
+++ b/tools/lint/python/compat.py
@@ -25,28 +25,38 @@ class PyCompatProcess(ProcessHandlerMixi
         self.config = config
         kwargs['processOutputLine'] = [self.process_line]
         ProcessHandlerMixin.__init__(self, *args, **kwargs)
 
     def process_line(self, line):
         try:
             res = json.loads(line)
         except ValueError:
-            print('Non JSON output from linter, will not be processed: {}'.format(line))
+            print('Non JSON output from {} linter: {}'.format(self.config['name'], line))
             return
 
         res['level'] = 'error'
         results.append(result.from_config(self.config, **res))
 
 
+def setup(python):
+    """Setup doesn't currently do any bootstrapping. For now, this function
+    is only used to print the warning message.
+    """
+    binary = find_executable(python)
+    if not binary:
+        # TODO Bootstrap python2/python3 if not available
+        print('warning: {} not detected, skipping py-compat check'.format(python))
+
+
 def run_linter(python, paths, config, **lintargs):
     binary = find_executable(python)
     if not binary:
-        # TODO bootstrap python3 if not available
-        print('error: {} not detected, aborting py-compat check'.format(python))
+        # If we're in automation, this is fatal. Otherwise, the warning in the
+        # setup method was already printed.
         if 'MOZ_AUTOMATION' in os.environ:
             return 1
         return []
 
     root = lintargs['root']
     pattern = "**/*.py"
     exclude = [mozpath.join(root, e) for e in lintargs.get('exclude', [])]
     files = []
@@ -72,14 +82,22 @@ def run_linter(python, paths, config, **
         try:
             proc.wait()
         except KeyboardInterrupt:
             proc.kill()
 
     return results
 
 
+def setuppy2(root):
+    return setup('python2')
+
+
 def lintpy2(*args, **kwargs):
     return run_linter('python2', *args, **kwargs)
 
 
+def setuppy3(root):
+    return setup('python3')
+
+
 def lintpy3(*args, **kwargs):
     return run_linter('python3', *args, **kwargs)