Bug 1385381 - Detect and expose Python 3 to the build system; r?chmanchester draft
authorGregory Szorc <gps@mozilla.com>
Thu, 27 Jul 2017 21:19:25 -0700
changeset 646208 47fae18460aa6a556ebab6e154edc65d6931bd27
parent 646207 824d4f269c6323e1ad2bd8ebeb6496d60b8ba3e5
child 726154 82744992b1a65123d6c8959c8002ddd89df4e319
push id74024
push userbmo:gps@mozilla.com
push dateMon, 14 Aug 2017 23:36:25 +0000
reviewerschmanchester
bugs1385381
milestone57.0a1
Bug 1385381 - Detect and expose Python 3 to the build system; r?chmanchester Various people want to start experimenting with Python 3 in the build system and in related tools (like mach). We want to make it easy to find and use an appropriate Python 3 binary. This commit introduces a generic function for finding a Python 3 binary and resolving its version. We use the new code in configure to set PYTHON3 and PYTHON3_VERSION subst entries for later consultation. We also expose a cached attribute on the base class used by many mach and build system types to return a Python 3 executable's info. By default, we only find Python 3.5+. From my experience, Python 3.5 was the first Python 3 where it was reasonable to write code that supports both Python 2 and 3 (mainly due to the restoration of the % operator on bytes types). We could probably support Python 3.4 in the build system. But for now I'd like to see if we can get away with 3.5+. MozReview-Commit-ID: BlwCJ3kkjY9
build/moz.configure/init.configure
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/pythonutil.py
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -302,16 +302,60 @@ def shell(value, mozillabuild):
     shell = 'sh'
     if mozillabuild:
         shell = mozillabuild[0] + '/msys/bin/sh'
     if sys.platform == 'win32':
         shell = shell + '.exe'
     return find_program(shell)
 
 
+# Python 3
+# ========
+
+option(env='PYTHON3', nargs=1, help='Python 3 interpreter (3.5 or later)')
+
+@depends('PYTHON3')
+@checking('for Python 3',
+          callback=lambda x: '%s (%s)' % (x.path, x.str_version) if x else 'no')
+@imports(_from='__builtin__', _import='Exception')
+@imports(_from='mozbuild.pythonutil', _import='find_python3_executable')
+@imports(_from='mozbuild.pythonutil', _import='python_executable_version')
+def python3(env_python):
+    python = env_python[0] if env_python else None
+
+    # If Python given by environment variable, it must work.
+    if python:
+        try:
+            version = python_executable_version(python).version
+        except Exception as e:
+            raise FatalCheckError('could not determine version of PYTHON '
+                                  '(%s): %s' % (python, e))
+
+        if version < (3, 5, 0):
+            raise FatalCheckError('PYTHON3 must point to Python 3.5 or newer; '
+                                  '%d.%d found' % (version[0], version[1]))
+    else:
+        # Fall back to the search routine.
+        python, version = find_python3_executable(min_version='3.5.0')
+
+        if not python:
+            return None
+
+        # The API returns a bytes whereas everything in configure is unicode.
+        python = python.decode('utf-8')
+
+    return namespace(
+        path=python,
+        version=version,
+        str_version='.'.join(str(v) for v in version),
+    )
+
+set_config('PYTHON3', depends_if(python3)(lambda p: p.path))
+set_config('PYTHON3_VERSION', depends_if(python3)(lambda p: p.str_version))
+
 # Source checkout and version control integration.
 # ================================================
 
 @depends(check_build_environment, 'MOZ_AUTOMATION', '--help')
 @checking('for vcs source checkout')
 @imports('os')
 def vcs_checkout_type(build_env, automation, _):
     if os.path.exists(os.path.join(build_env.topsrcdir, '.hg')):
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -8,27 +8,27 @@ import json
 import logging
 import mozpack.path as mozpath
 import multiprocessing
 import os
 import subprocess
 import sys
 import which
 
-from mach.mixin.logging import LoggingMixin
 from mach.mixin.process import ProcessExecutionMixin
 from mozversioncontrol import get_repository_object
 
 from .backend.configenvironment import ConfigEnvironment
 from .controller.clobber import Clobberer
 from .mozconfig import (
     MozconfigFindException,
     MozconfigLoadException,
     MozconfigLoader,
 )
+from .pythonutil import find_python3_executable
 from .util import memoized_property
 from .virtualenv import VirtualenvManager
 
 
 _config_guess_output = []
 
 
 def ancestors(path):
@@ -283,16 +283,35 @@ class MozbuildObject(ProcessExecutionMix
         return env
 
     @memoized_property
     def repository(self):
         '''Get a `mozversioncontrol.Repository` object for the
         top source directory.'''
         return get_repository_object(self.topsrcdir)
 
+    @memoized_property
+    def python3(self):
+        """Obtain info about a Python 3 executable.
+
+        Returns a tuple of an executable path and its version (as a tuple).
+        Either both entries will have a value or both will be None.
+        """
+        # Search configured build info first. Then fall back to system.
+        try:
+            subst = self.substs
+
+            if 'PYTHON3' in subst:
+                version = tuple(map(int, subst['PYTHON3_VERSION'].split('.')))
+                return subst['PYTHON3'], version
+        except BuildEnvironmentNotFoundException:
+            pass
+
+        return find_python3_executable()
+
     def is_clobber_needed(self):
         if not os.path.exists(self.topobjdir):
             return False
         return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed()
 
     def get_binary_path(self, what='app', validate_exists=True, where='default'):
         """Obtain the path to a compiled binary for this build configuration.
 
--- a/python/mozbuild/mozbuild/pythonutil.py
+++ b/python/mozbuild/mozbuild/pythonutil.py
@@ -1,25 +1,98 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import os
+import subprocess
 import sys
 
+from distutils.version import (
+    StrictVersion,
+)
+
 
 def iter_modules_in_path(*paths):
     paths = [os.path.abspath(os.path.normcase(p)) + os.sep
              for p in paths]
     for name, module in sys.modules.items():
         if not hasattr(module, '__file__'):
             continue
 
         path = module.__file__
 
         if path.endswith('.pyc'):
             path = path[:-1]
         path = os.path.abspath(os.path.normcase(path))
 
         if any(path.startswith(p) for p in paths):
             yield path
+
+
+def python_executable_version(exe):
+    """Determine the version of a Python executable by invoking it.
+
+    May raise ``subprocess.CalledProcessError`` or ``ValueError`` on failure.
+    """
+    program = "import sys; print('.'.join(map(str, sys.version_info[0:3])))"
+    out = subprocess.check_output([exe, '-c', program]).rstrip()
+    return StrictVersion(out)
+
+
+def find_python3_executable(min_version='3.5.0'):
+    """Find a Python 3 executable.
+
+    Returns a tuple containing the the path to an executable binary and a
+    version tuple. Both tuple entries will be None if a Python executable
+    could not be resolved.
+    """
+    import which
+
+    if not min_version.startswith('3.'):
+        raise ValueError('min_version expected a 3.x string, got %s' %
+                         min_version)
+
+    min_version = StrictVersion(min_version)
+
+    if sys.version_info.major >= 3:
+        our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3]))
+
+        if our_version >= min_version:
+            # This will potentially return a virtualenv Python. It's probably
+            # OK for now...
+            return sys.executable, our_version.version
+
+        # Else fall back to finding another binary.
+
+    # https://www.python.org/dev/peps/pep-0394/ defines how the Python binary
+    # should be named. `python3` should refer to some Python 3. `python` may
+    # refer to a Python 2 or 3. `pythonX.Y` may exist.
+    #
+    # Since `python` is ambiguous and `python3` should always exist, we
+    # ignore `python` here. We instead look for the preferred `python3` first
+    # and fall back to `pythonX.Y` if it isn't found or doesn't meet our
+    # version requirements.
+    names = ['python3']
+
+    # Look for `python3.Y` down to our minimum version.
+    for minor in range(9, min_version.version[1] - 1, -1):
+        names.append('python3.%d' % minor)
+
+    for name in names:
+        try:
+            exe = which.which(name)
+        except which.WhichError:
+            continue
+
+        # We always verify we can invoke the executable and its version is
+        # sane.
+        try:
+            version = python_executable_version(exe)
+        except (subprocess.CalledProcessError, ValueError):
+            continue
+
+        if version >= min_version:
+            return exe, version.version
+
+    return None, None