--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -2,22 +2,24 @@
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
'''Python usage, esp. virtualenv.
'''
+import distutils.version
import os
import subprocess
import sys
import time
import json
import traceback
+import urlparse
import mozharness
from mozharness.base.script import (
PostScriptAction,
PostScriptRun,
PreScriptAction,
PreScriptRun,
)
@@ -252,20 +254,30 @@ class VirtualenvMixin(object):
# the 'install' in the executable name hits UAC
# - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
# - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
default = [self.query_python_path(), self.query_python_path('easy_install-script.py')]
command = self.query_exe('easy_install', default=default, return_type="list")
else:
self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
- # Add --find-links pages to look at
+ # Add --find-links pages to look at. Add --trusted-host automatically if
+ # the host isn't secure. This allows modern versions of pip to connect
+ # without requiring an override.
proxxy = Proxxy(self.config, self.log_obj)
+ trusted_hosts = set()
for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
command.extend(["--find-links", link])
+ parsed = urlparse.urlparse(link)
+ if parsed.scheme != 'https':
+ trusted_hosts.add(parsed.hostname)
+
+ if self.pip_version >= distutils.version.LooseVersion('6.0'):
+ for host in sorted(trusted_hosts):
+ command.extend(['--trusted-host', host])
# module_url can be None if only specifying requirements files
if module_url:
if editable:
if install_method in (None, 'pip'):
command += ['-e']
else:
self.fatal("editable installs not supported for install_method %s" % install_method)
@@ -344,55 +356,85 @@ class VirtualenvMixin(object):
'/path/to/requirements1.txt',
'/path/to/requirements2.txt'
]
"""
c = self.config
dirs = self.query_abs_dirs()
venv_path = self.query_virtualenv_path()
self.info("Creating virtualenv %s" % venv_path)
- virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
- if isinstance(virtualenv, str):
- # allow for [python, virtualenv] in config
- virtualenv = [virtualenv]
- if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
- self.add_summary("The executable '%s' is not found; not creating "
- "virtualenv!" % virtualenv[0], level=FATAL)
- return -1
+ # If running from a source checkout, use the virtualenv that is
+ # vendored since that is deterministic.
+ if self.topsrcdir:
+ virtualenv = [
+ sys.executable,
+ os.path.join(self.topsrcdir, 'python', 'virtualenv', 'virtualenv.py')
+ ]
+ virtualenv_options = c.get('virtualenv_options', [])
+ # Don't create symlinks. If we don't do this, permissions issues may
+ # hinder virtualenv creation or operation. Ideally we should do this
+ # below when using the system virtualenv. However, this is a newer
+ # feature and isn't guaranteed to be supported.
+ virtualenv_options.append('--always-copy')
+
+ # No source checkout. Try to find virtualenv from config options
+ # or search path.
+ else:
+ virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
+ if isinstance(virtualenv, str):
+ # allow for [python, virtualenv] in config
+ virtualenv = [virtualenv]
- # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
- # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
- if c.get('virtualenv_python_dll'):
- # We may someday want to copy a differently-named dll, but
- # let's not think about that right now =\
- dll_name = os.path.basename(c['virtualenv_python_dll'])
- target = self.query_python_path(dll_name)
- scripts_dir = os.path.dirname(target)
- self.mkdir_p(scripts_dir)
- self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
- else:
- self.mkdir_p(dirs['abs_work_dir'])
+ if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
+ self.add_summary("The executable '%s' is not found; not creating "
+ "virtualenv!" % virtualenv[0], level=FATAL)
+ return -1
- # make this list configurable?
- for module in ('distribute', 'pip'):
- if c.get('%s_url' % module):
- self.download_file(c['%s_url' % module],
- parent_dir=dirs['abs_work_dir'])
+ # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
+ # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
+ if c.get('virtualenv_python_dll'):
+ # We may someday want to copy a differently-named dll, but
+ # let's not think about that right now =\
+ dll_name = os.path.basename(c['virtualenv_python_dll'])
+ target = self.query_python_path(dll_name)
+ scripts_dir = os.path.dirname(target)
+ self.mkdir_p(scripts_dir)
+ self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
+ else:
+ self.mkdir_p(dirs['abs_work_dir'])
- virtualenv_options = c.get('virtualenv_options',
- ['--no-site-packages', '--distribute'])
+ # make this list configurable?
+ for module in ('distribute', 'pip'):
+ if c.get('%s_url' % module):
+ self.download_file(c['%s_url' % module],
+ parent_dir=dirs['abs_work_dir'])
+
+ virtualenv_options = c.get('virtualenv_options',
+ ['--no-site-packages', '--distribute'])
if os.path.exists(self.query_python_path()):
self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path())
else:
self.run_command(virtualenv + virtualenv_options + [venv_path],
cwd=dirs['abs_work_dir'],
error_list=VirtualenvErrorList,
halt_on_failure=True)
+
+ # Resolve the pip version so we can conditionally do things if we have
+ # a modern pip.
+ pip = self.query_python_path('pip')
+ output = self.get_output_from_command([pip, '--version'],
+ halt_on_failure=True)
+ words = output.split()
+ if words[0] != 'pip':
+ self.fatal('pip --version output is weird: %s' % output)
+ pip_version = words[1]
+ self.pip_version = distutils.version.LooseVersion(pip_version)
+
if not modules:
modules = c.get('virtualenv_modules', [])
if not requirements:
requirements = c.get('virtualenv_requirements', [])
if not modules and requirements:
self.install_module(requirements=requirements,
install_method='pip')
for module in modules: