Bug 1385381 - Detect and expose Python 3 to the build system; r?chmanchester
Various people want to start experimenting with Python 3 in the build
system and in related tools (like mach).
We want to make it easy to find and use an appropriate Python 3
binary.
This commit introduces a generic function for finding a Python 3
binary and resolving its version.
We use the new code in configure to set PYTHON3 and PYTHON3_VERSION
subst entries for later consultation.
We also expose a cached attribute on the base class used by many
mach and build system types to return a Python 3 executable's info.
By default, we only find Python 3.5+. From my experience, Python 3.5
was the first Python 3 where it was reasonable to write code that
supports both Python 2 and 3 (mainly due to the restoration of the
% operator on bytes types). We could probably support Python 3.4
in the build system. But for now I'd like to see if we can get
away with 3.5+.
MozReview-Commit-ID: BlwCJ3kkjY9
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -302,16 +302,60 @@ def shell(value, mozillabuild):
shell = 'sh'
if mozillabuild:
shell = mozillabuild[0] + '/msys/bin/sh'
if sys.platform == 'win32':
shell = shell + '.exe'
return find_program(shell)
+# Python 3
+# ========
+
+option(env='PYTHON3', nargs=1, help='Python 3 interpreter (3.5 or later)')
+
+@depends('PYTHON3')
+@checking('for Python 3',
+ callback=lambda x: '%s (%s)' % (x.path, x.str_version) if x else 'no')
+@imports(_from='__builtin__', _import='Exception')
+@imports(_from='mozbuild.pythonutil', _import='find_python3_executable')
+@imports(_from='mozbuild.pythonutil', _import='python_executable_version')
+def python3(env_python):
+ python = env_python[0] if env_python else None
+
+ # If Python given by environment variable, it must work.
+ if python:
+ try:
+ version = python_executable_version(python).version
+ except Exception as e:
+ raise FatalCheckError('could not determine version of PYTHON '
+ '(%s): %s' % (python, e))
+
+ if version < (3, 5, 0):
+ raise FatalCheckError('PYTHON3 must point to Python 3.5 or newer; '
+ '%d.%d found' % (version[0], version[1]))
+ else:
+ # Fall back to the search routine.
+ python, version = find_python3_executable(min_version='3.5.0')
+
+ if not python:
+ return None
+
+ # The API returns a bytes whereas everything in configure is unicode.
+ python = python.decode('utf-8')
+
+ return namespace(
+ path=python,
+ version=version,
+ str_version='.'.join(str(v) for v in version),
+ )
+
+set_config('PYTHON3', depends_if(python3)(lambda p: p.path))
+set_config('PYTHON3_VERSION', depends_if(python3)(lambda p: p.str_version))
+
# Source checkout and version control integration.
# ================================================
@depends(check_build_environment, 'MOZ_AUTOMATION', '--help')
@checking('for vcs source checkout')
@imports('os')
def vcs_checkout_type(build_env, automation, _):
if os.path.exists(os.path.join(build_env.topsrcdir, '.hg')):
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -8,27 +8,27 @@ import json
import logging
import mozpack.path as mozpath
import multiprocessing
import os
import subprocess
import sys
import which
-from mach.mixin.logging import LoggingMixin
from mach.mixin.process import ProcessExecutionMixin
from mozversioncontrol import get_repository_object
from .backend.configenvironment import ConfigEnvironment
from .controller.clobber import Clobberer
from .mozconfig import (
MozconfigFindException,
MozconfigLoadException,
MozconfigLoader,
)
+from .pythonutil import find_python3_executable
from .util import memoized_property
from .virtualenv import VirtualenvManager
_config_guess_output = []
def ancestors(path):
@@ -283,16 +283,35 @@ class MozbuildObject(ProcessExecutionMix
return env
@memoized_property
def repository(self):
'''Get a `mozversioncontrol.Repository` object for the
top source directory.'''
return get_repository_object(self.topsrcdir)
+ @memoized_property
+ def python3(self):
+ """Obtain info about a Python 3 executable.
+
+ Returns a tuple of an executable path and its version (as a tuple).
+ Either both entries will have a value or both will be None.
+ """
+ # Search configured build info first. Then fall back to system.
+ try:
+ subst = self.substs
+
+ if 'PYTHON3' in subst:
+ version = tuple(map(int, subst['PYTHON3_VERSION'].split('.')))
+ return subst['PYTHON3'], version
+ except BuildEnvironmentNotFoundException:
+ pass
+
+ return find_python3_executable()
+
def is_clobber_needed(self):
if not os.path.exists(self.topobjdir):
return False
return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed()
def get_binary_path(self, what='app', validate_exists=True, where='default'):
"""Obtain the path to a compiled binary for this build configuration.
--- a/python/mozbuild/mozbuild/pythonutil.py
+++ b/python/mozbuild/mozbuild/pythonutil.py
@@ -1,25 +1,98 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import
import os
+import subprocess
import sys
+from distutils.version import (
+ StrictVersion,
+)
+
def iter_modules_in_path(*paths):
paths = [os.path.abspath(os.path.normcase(p)) + os.sep
for p in paths]
for name, module in sys.modules.items():
if not hasattr(module, '__file__'):
continue
path = module.__file__
if path.endswith('.pyc'):
path = path[:-1]
path = os.path.abspath(os.path.normcase(path))
if any(path.startswith(p) for p in paths):
yield path
+
+
+def python_executable_version(exe):
+ """Determine the version of a Python executable by invoking it.
+
+ May raise ``subprocess.CalledProcessError`` or ``ValueError`` on failure.
+ """
+ program = "import sys; print('.'.join(map(str, sys.version_info[0:3])))"
+ out = subprocess.check_output([exe, '-c', program]).rstrip()
+ return StrictVersion(out)
+
+
+def find_python3_executable(min_version='3.5.0'):
+ """Find a Python 3 executable.
+
+ Returns a tuple containing the the path to an executable binary and a
+ version tuple. Both tuple entries will be None if a Python executable
+ could not be resolved.
+ """
+ import which
+
+ if not min_version.startswith('3.'):
+ raise ValueError('min_version expected a 3.x string, got %s' %
+ min_version)
+
+ min_version = StrictVersion(min_version)
+
+ if sys.version_info.major >= 3:
+ our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3]))
+
+ if our_version >= min_version:
+ # This will potentially return a virtualenv Python. It's probably
+ # OK for now...
+ return sys.executable, our_version.version
+
+ # Else fall back to finding another binary.
+
+ # https://www.python.org/dev/peps/pep-0394/ defines how the Python binary
+ # should be named. `python3` should refer to some Python 3. `python` may
+ # refer to a Python 2 or 3. `pythonX.Y` may exist.
+ #
+ # Since `python` is ambiguous and `python3` should always exist, we
+ # ignore `python` here. We instead look for the preferred `python3` first
+ # and fall back to `pythonX.Y` if it isn't found or doesn't meet our
+ # version requirements.
+ names = ['python3']
+
+ # Look for `python3.Y` down to our minimum version.
+ for minor in range(9, min_version.version[1] - 1, -1):
+ names.append('python3.%d' % minor)
+
+ for name in names:
+ try:
+ exe = which.which(name)
+ except which.WhichError:
+ continue
+
+ # We always verify we can invoke the executable and its version is
+ # sane.
+ try:
+ version = python_executable_version(exe)
+ except (subprocess.CalledProcessError, ValueError):
+ continue
+
+ if version >= min_version:
+ return exe, version.version
+
+ return None, None