Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r?ahal
MozReview-Commit-ID: AzmdDgAgZgI
--- a/Pipfile
+++ b/Pipfile
@@ -11,12 +11,8 @@ jsmin = "==2.1.0"
json-e = "==2.5.0"
pipenv = "==2018.5.18"
pytest = "==3.2.5"
python-hglib = "==2.4"
requests = "==2.9.1"
six = "==1.10.0"
virtualenv = "==15.2.0"
voluptuous = "==0.10.5"
-
-
-[requires]
-python_version = "2.7"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,17 +1,15 @@
{
"_meta": {
"hash": {
- "sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba"
+ "sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f"
},
"pipfile-spec": 6,
- "requires": {
- "python_version": "2.7"
- },
+ "requires": {},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
deleted file mode 100644
--- a/python/Pipfile
+++ /dev/null
@@ -1,36 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-"d5b4a14" = {path = "./mach"}
-"8ddb376" = {path = "./mozbuild"}
-"b3ddbcf" = {path = "./mozterm"}
-"38a4a9a" = {path = "./mozversioncontrol"}
-"26d92fb" = {path = "./../config/mozunit"}
-"cea2946" = {path = "./../testing/mozbase/manifestparser"}
-"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
-"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
-"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
-"58d0848" = {path = "./../testing/mozbase/mozfile"}
-"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
-"7329809" = {path = "./../testing/mozbase/mozinfo"}
-"501835d" = {path = "./../testing/mozbase/mozinstall"}
-"807c1c5" = {path = "./../testing/mozbase/mozlog"}
-"e09e103" = {path = "./../testing/mozbase/moznetwork"}
-"132adec" = {path = "./../testing/mozbase/mozprocess"}
-"d88f467" = {path = "./../testing/mozbase/mozprofile"}
-"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
-"6477f20" = {path = "./../testing/mozbase/moztest"}
-"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
-"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
-"110bcc4" = {path = "./../third_party/python/jsmin"}
-"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
-"c2c21d9" = {path = "./../third_party/python/py"}
-"f4b00e9" = {path = "./../third_party/python/pytest"}
-"053111f" = {path = "./../third_party/python/requests"}
-"d250320" = {path = "./../third_party/python/six"}
-"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
-
-[dev-packages]
deleted file mode 100644
--- a/python/Pipfile.lock
+++ /dev/null
@@ -1,106 +0,0 @@
-{
- "_meta": {
- "hash": {
- "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
- },
- "pipfile-spec": 6,
- "requires": {},
- "sources": [
- {
- "name": "pypi",
- "url": "https://pypi.org/simple",
- "verify_ssl": true
- }
- ]
- },
- "default": {
- "053111f": {
- "path": "./../third_party/python/requests"
- },
- "110bcc4": {
- "path": "./../third_party/python/jsmin"
- },
- "132adec": {
- "path": "./../testing/mozbase/mozprocess"
- },
- "195ae2e": {
- "path": "./../testing/mozbase/mozdebug"
- },
- "1de94f2": {
- "path": "./../testing/mozbase/mozrunner"
- },
- "26d92fb": {
- "path": "./../config/mozunit"
- },
- "38a4a9a": {
- "path": "./mozversioncontrol"
- },
- "47200d8": {
- "markers": "python_version < '3'",
- "path": "./../third_party/python/futures"
- },
- "501835d": {
- "path": "./../testing/mozbase/mozinstall"
- },
- "58d0848": {
- "path": "./../testing/mozbase/mozfile"
- },
- "6477f20": {
- "path": "./../testing/mozbase/moztest"
- },
- "7329809": {
- "path": "./../testing/mozbase/mozinfo"
- },
- "807c1c5": {
- "path": "./../testing/mozbase/mozlog"
- },
- "8dab59a": {
- "path": "./../testing/mozbase/mozdevice"
- },
- "8ddb376": {
- "path": "./mozbuild"
- },
- "b3ddbcf": {
- "path": "./mozterm"
- },
- "c2c21d9": {
- "path": "./../third_party/python/py"
- },
- "c49d32a": {
- "markers": "python_version < '3.3'",
- "path": "./../third_party/python/mock-1.0.0"
- },
- "cea2946": {
- "path": "./../testing/mozbase/manifestparser"
- },
- "d250320": {
- "path": "./../third_party/python/six"
- },
- "d5b4a14": {
- "path": "./mach"
- },
- "d88f467": {
- "path": "./../testing/mozbase/mozprofile"
- },
- "e09e103": {
- "path": "./../testing/mozbase/moznetwork"
- },
- "f1d74ca": {
- "path": "./../testing/mozbase/mozversion"
- },
- "f1de77a": {
- "markers": "python_version < '3.3'",
- "path": "./../third_party/python/which"
- },
- "f4b00e9": {
- "path": "./../third_party/python/pytest"
- },
- "fd0b608": {
- "path": "./../testing/mozbase/mozhttpd"
- },
- "ffcf6e6": {
- "path": "./../testing/mozbase/mozcrash"
- }
- },
- "develop": {}
-}
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -94,21 +94,18 @@ class MachCommands(MachCommandBase):
def run_python_tests(self,
tests=None,
test_objects=None,
subsuite=None,
verbose=False,
jobs=1,
three=False,
**kwargs):
- if three:
- # use pipenv to run tests against Python 3
- self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
- else:
- self._activate_virtualenv()
+ pipenv_args = ['--three' if three else '--two']
+ self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True)
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor='python')
else:
# We've received test_objects from |mach test|. We need to ignore
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -747,26 +747,26 @@ class MozbuildObject(ProcessExecutionMix
def _set_log_level(self, verbose):
self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
def ensure_pipenv(self):
self._activate_virtualenv()
pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
if not os.path.exists(pipenv):
- pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
- self.virtualenv_manager.install_pip_requirements(
- pipenv_reqs, require_hashes=False, vendored=True)
+ for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']:
+ path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package))
+ self.virtualenv_manager.install_pip_package(path, vendored=True)
return pipenv
- def activate_pipenv(self, path, args=None):
- if not os.path.exists(path):
- raise Exception('Pipfile not found: %s.' % path)
+ def activate_pipenv(self, pipfile=None, args=None, populate=False):
+ if pipfile is not None and not os.path.exists(pipfile):
+ raise Exception('Pipfile not found: %s.' % pipfile)
self.ensure_pipenv()
- self.virtualenv_manager.activate_pipenv(path, args)
+ self.virtualenv_manager.activate_pipenv(pipfile, args, populate)
class MachCommandBase(MozbuildObject):
"""Base class for mach command providers that wish to be MozbuildObjects.
This provides a level of indirection so MozbuildObject can be refactored
without having to change everything that inherits from it.
"""
deleted file mode 100644
--- a/python/mozbuild/mozbuild/pipenv.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-third_party/python/certifi
-third_party/python/pipenv
-third_party/python/six
-third_party/python/virtualenv
-third_party/python/virtualenv-clone
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -15,34 +15,38 @@ import sys
import warnings
from distutils.version import LooseVersion
IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/')
IS_CYGWIN = (sys.platform == 'cygwin')
-# Minimum version of Python required to build.
-MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3')
-MINIMUM_PYTHON_MAJOR = 2
+# Minimum versions of Python required to build.
+MINIMUM_PYTHON_VERSIONS = {
+ 2: LooseVersion('2.7.3'),
+ 3: LooseVersion('3.5.0')
+}
UPGRADE_WINDOWS = '''
Please upgrade to the latest MozillaBuild development environment. See
https://developer.mozilla.org/en-US/docs/Developer_Guide/Build_Instructions/Windows_Prerequisites
'''.lstrip()
UPGRADE_OTHER = '''
Run |mach bootstrap| to ensure your system is up to date.
If you still receive this error, your shell environment is likely detecting
another Python version. Ensure a modern Python can be found in the paths
defined by the $PATH environment variable and try again.
'''.lstrip()
+here = os.path.abspath(os.path.dirname(__file__))
+
class VirtualenvManager(object):
"""Contains logic for managing virtualenvs for building the tree."""
def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle,
manifest_path):
"""Create a new manager.
@@ -202,17 +206,17 @@ class VirtualenvManager(object):
raise Exception(
'Failed to create virtualenv: %s' % self.virtualenv_root)
self.write_exe_info(python)
return self.virtualenv_root
def packages(self):
- with file(self.manifest_path, 'rU') as fh:
+ with open(self.manifest_path, 'rU') as fh:
packages = [line.rstrip().split(':')
for line in fh]
return packages
def populate(self):
"""Populate the virtualenv.
The manifest file consists of colon-delimited fields. The first field
@@ -245,17 +249,16 @@ class VirtualenvManager(object):
search path. e.g. "objdir:build" will add $topobjdir/build to the
search path.
Note that the Python interpreter running this function should be the
one from the virtualenv. If it is the system Python or if the
environment is not configured properly, packages could be installed
into the wrong place. This is how virtualenv's work.
"""
-
packages = self.packages()
python_lib = distutils.sysconfig.get_python_lib()
def handle_package(package):
if package[0] == 'setup.py':
assert len(package) >= 2
self.call_setup(os.path.join(self.topsrcdir, package[1]),
@@ -460,37 +463,46 @@ class VirtualenvManager(object):
virtualenv, you can simply instantiate an instance of this class
and call .ensure() and .activate() to make the virtualenv active.
"""
execfile(self.activate_path, dict(__file__=self.activate_path))
if isinstance(os.environ['PATH'], unicode):
os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
- def install_pip_package(self, package):
+ def install_pip_package(self, package, vendored=False):
"""Install a package via pip.
The supplied package is specified using a pip requirement specifier.
e.g. 'foo' or 'foo==1.0'.
If the package is already installed, this is a no-op.
+
+ If vendored is True, no package index will be used and no dependencies
+ will be installed.
"""
from pip.req import InstallRequirement
req = InstallRequirement.from_line(package)
req.check_if_exists()
if req.satisfied_by is not None:
return
args = [
'install',
'--use-wheel',
package,
]
+ if vendored:
+ args.extend([
+ '--no-deps',
+ '--no-index',
+ ])
+
return self._run_pip(args)
def install_pip_requirements(self, path, require_hashes=True, quiet=False, vendored=False):
"""Install a pip requirements.txt file.
The supplied path is a text file containing pip requirement
specifiers.
@@ -525,52 +537,72 @@ class VirtualenvManager(object):
def _run_pip(self, args):
# It's tempting to call pip natively via pip.main(). However,
# the current Python interpreter may not be the virtualenv python.
# This will confuse pip and cause the package to attempt to install
# against the executing interpreter. By creating a new process, we
# force the virtualenv's interpreter to be used and all is well.
# It /might/ be possible to cheat and set sys.executable to
# self.python_path. However, this seems more risk than it's worth.
- subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
- stderr=subprocess.STDOUT)
+ pip = os.path.join(self.bin_path, 'pip')
+ subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir)
+
+ def activate_pipenv(self, pipfile=None, args=None, populate=False):
+ """Activate a virtual environment managed by pipenv
- def activate_pipenv(self, pipfile, args=None):
- """Install a Pipfile located at path and activate environment"""
+ If ``pipfile`` is not ``None`` then the Pipfile located at the path
+ provided will be used to create the virtual environment. If
+ ``populate`` is ``True`` then the virtual environment will be
+ populated from the manifest file. The optional ``args`` list will be
+ passed to the pipenv commands.
+ """
pipenv = os.path.join(self.bin_path, 'pipenv')
env = os.environ.copy()
env.update({
- 'PIPENV_IGNORE_VIRTUALENVS': '1',
- 'PIPENV_PIPFILE': pipfile,
- 'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
+ b'PIPENV_IGNORE_VIRTUALENVS': b'1',
+ b'WORKON_HOME': str(os.path.normpath(os.path.join(self.topobjdir, '_virtualenvs'))),
})
args = args or []
+
+ if pipfile is not None:
+ # Install from Pipfile
+ env[b'PIPENV_PIPFILE'] = str(pipfile)
+ args.append('install')
+
subprocess.check_call(
- [pipenv, 'install'] + args,
+ [pipenv] + args,
stderr=subprocess.STDOUT,
env=env)
self.virtualenv_root = subprocess.check_output(
[pipenv, '--venv'],
stderr=subprocess.STDOUT,
env=env).rstrip()
+ if populate:
+ # Populate from the manifest
+ subprocess.check_call([
+ pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+ self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path],
+ stderr=subprocess.STDOUT, env=env)
+
self.activate()
def verify_python_version(log_handle):
"""Ensure the current version of Python is sufficient."""
major, minor, micro = sys.version_info[:3]
our = LooseVersion('%d.%d.%d' % (major, minor, micro))
- if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION:
- log_handle.write('Python %s or greater (but not Python 3) is '
- 'required to build. ' % MINIMUM_PYTHON_VERSION)
+ if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]:
+ log_handle.write('One of the following Python versions are required to build:\n')
+ for minver in MINIMUM_PYTHON_VERSIONS.values():
+ log_handle.write('* Python %s or greater\n' % minver)
log_handle.write('You are running Python %s.\n' % our)
if os.name in ('nt', 'ce'):
log_handle.write(UPGRADE_WINDOWS)
else:
log_handle.write(UPGRADE_OTHER)
sys.exit(1)