Bug 1361732 - Mozharness support for a second Python3 virtualenv draft
authorArmen Zambrano G. <armenzg@mozilla.com>
Fri, 05 May 2017 15:03:05 -0400
changeset 573820 996e64c35656c0605cbcb308350b15f2666f3691
parent 573811 17d8a1e278a9c54a6fdda9d390abce4077e55b20
child 573821 270ab0b73ab7f9ca3fbf8fd8a7006857880dd11f
push id57517
push userarmenzg@mozilla.com
push dateMon, 08 May 2017 01:17:17 +0000
bugs1361732
milestone55.0a1
Bug 1361732 - Mozharness support for a second Python3 virtualenv Currently Mozharness creates a Python 2.7 virtualenvironment with all the packages needed for Mozharness to run plus any applications that need to execute within it. For Talos, we now need a second Python virtual environment for mitmproxy, however, this package is for Python 3 and not for Python 2 (perhaps it could have worked). In any case, having the ability to support create Python 3 virtual environments will be handy. Specially since the approach of creating virtual environments is not by using 'virtualenv /path/to/venv' but by calling the venv module like 'python3 -m venv /path/to/venv'. The initial implementation only supports a list of modules. The following iteration will support requirement files and other parameters needed for pip and internal pypi hosts. MozReview-Commit-ID: 82hcPsDTgaz
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -792,14 +792,71 @@ class ResourceMonitoringMixin(Perfherder
             start_time, end_time = rm.phases[phase]
             cpu_percent, cpu_times, io, swap = resources(phase)
             log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
 
     def _tinderbox_print(self, message):
         self.info('TinderboxPrint: %s' % message)
 
 
+# This needs to be inherited only if you have already inherited ScriptMixin
+class Python3Virtualenv(object):
+    ''' Support Python3.5+ virtualenv creation.'''
+    py3_initialized_venv = False
+
+    def py3_venv_configuration(self, python_path, venv_path):
+        '''We don't use __init__ to allow integrating with other mixins.
+
+        python_path - Path to Python 3 binary.
+        venv_path - Path to virtual environment to be created.
+        '''
+        self.py3_initialized_venv = True
+        self.py3_python_path = os.path.abspath(python_path)
+        version = self.get_output_from_command(
+                    [self.py3_python_path, '--version']).split()[-1]
+        # Using -m venv is only used on 3.5+ versions
+        assert version > '3.5.0'
+        self.py3_venv_path = os.path.abspath(venv_path)
+        self.py3_pip_path = os.path.join(self.py3_path_to_executables(), 'pip')
+
+    def py3_path_to_executables(self):
+        platform = self.platform_name()
+        if platform.startswith('win'):
+            return os.path.join(self.py3_venv_path, 'Scripts')
+        else:
+            return os.path.join(self.py3_venv_path, 'bin')
+
+    def py3_venv_initialized(func):
+        def call(self, *args, **kwargs):
+            if not self.py3_initialized_venv:
+                raise Exception('You need to call py3_venv_configuration() '
+                                'before using this method.')
+            func(self, *args, **kwargs)
+        return call
+
+    @py3_venv_initialized
+    def py3_create_venv(self):
+        '''Create Python environment with python3 -m venv /path/to/venv.'''
+        if os.path.exists(self.py3_venv_path):
+            self.info("Virtualenv %s appears to already exist; skipping "
+                      "virtualenv creation." % self.py3_venv_path)
+        else:
+            self.info('Running command...')
+            self.run_command(
+                '%s -m venv %s' % (self.py3_python_path, self.py3_venv_path),
+                error_list=VirtualenvErrorList,
+                halt_on_failure=True)
+
+    @py3_venv_initialized
+    def py3_install_modules(self, modules):
+        if not os.path.exists(self.py3_venv_path):
+            raise Exception('You need to call py3_create_venv() first.')
+
+        for m in modules:
+            self.run_command('%s install %s' % (self.py3_pip_path, m))
+
+
 # __main__ {{{1
 
 if __name__ == '__main__':
     '''TODO: unit tests.
     '''
     pass