Bug 1361732 - Mozharness support for a second Python3 virtualenv
Currently Mozharness creates a Python 2.7 virtualenvironment with all
the packages needed for Mozharness to run plus any applications that
need to execute within it.
For Talos, we now need a second Python virtual environment for
mitmproxy, however, this package is for Python 3 and not
for Python 2 (perhaps it could have worked).
In any case, having the ability to support create Python 3
virtual environments will be handy. Specially since the approach
of creating virtual environments is not by using 'virtualenv /path/to/venv'
but by calling the venv module like 'python3 -m venv /path/to/venv'.
The initial implementation only supports a list of modules.
The following iteration will support requirement files and other
parameters needed for pip and internal pypi hosts.
MozReview-Commit-ID: 82hcPsDTgaz
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -792,14 +792,71 @@ class ResourceMonitoringMixin(Perfherder
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
+# This needs to be inherited only if you have already inherited ScriptMixin
+class Python3Virtualenv(object):
+ ''' Support Python3.5+ virtualenv creation.'''
+ py3_initialized_venv = False
+
+ def py3_venv_configuration(self, python_path, venv_path):
+ '''We don't use __init__ to allow integrating with other mixins.
+
+ python_path - Path to Python 3 binary.
+ venv_path - Path to virtual environment to be created.
+ '''
+ self.py3_initialized_venv = True
+ self.py3_python_path = os.path.abspath(python_path)
+ version = self.get_output_from_command(
+ [self.py3_python_path, '--version']).split()[-1]
+ # Using -m venv is only used on 3.5+ versions
+ assert version > '3.5.0'
+ self.py3_venv_path = os.path.abspath(venv_path)
+ self.py3_pip_path = os.path.join(self.py3_path_to_executables(), 'pip')
+
+ def py3_path_to_executables(self):
+ platform = self.platform_name()
+ if platform.startswith('win'):
+ return os.path.join(self.py3_venv_path, 'Scripts')
+ else:
+ return os.path.join(self.py3_venv_path, 'bin')
+
+ def py3_venv_initialized(func):
+ def call(self, *args, **kwargs):
+ if not self.py3_initialized_venv:
+ raise Exception('You need to call py3_venv_configuration() '
+ 'before using this method.')
+ func(self, *args, **kwargs)
+ return call
+
+ @py3_venv_initialized
+ def py3_create_venv(self):
+ '''Create Python environment with python3 -m venv /path/to/venv.'''
+ if os.path.exists(self.py3_venv_path):
+ self.info("Virtualenv %s appears to already exist; skipping "
+ "virtualenv creation." % self.py3_venv_path)
+ else:
+ self.info('Running command...')
+ self.run_command(
+ '%s -m venv %s' % (self.py3_python_path, self.py3_venv_path),
+ error_list=VirtualenvErrorList,
+ halt_on_failure=True)
+
+ @py3_venv_initialized
+ def py3_install_modules(self, modules):
+ if not os.path.exists(self.py3_venv_path):
+ raise Exception('You need to call py3_create_venv() first.')
+
+ for m in modules:
+ self.run_command('%s install %s' % (self.py3_pip_path, m))
+
+
# __main__ {{{1
if __name__ == '__main__':
'''TODO: unit tests.
'''
pass