Bug 1388013 - Support running |mach python-test| against Python 3 using pipenv; r?ahal draft
authorDave Hunt <dhunt@mozilla.com>
Thu, 03 May 2018 10:34:22 +0100
changeset 802113 68610b0f5d4a9303f5de75ce6094d7a402cabff7
parent 802112 d86d781e8c92a276b3084e8aec661f982b62f151
child 802114 36a80d720f7370cb2698e8ffe4a1f47cfa56f9ba
push id111822
push userbmo:dave.hunt@gmail.com
push dateThu, 31 May 2018 11:50:12 +0000
reviewersahal
bugs1388013
milestone62.0a1
Bug 1388013 - Support running |mach python-test| against Python 3 using pipenv; r?ahal This patch allows executing |mach python-test| against Python 3 by specifying the optional |--three| command line option. When this option is present, pipenv will be used to manage a virtual environment using Python 3 and attempt to run the tests. When it is not present, pipenv will not be used, and everything will work as it did before this patch. My original plan was to use pipenv regardless of the target version of Python, however I encountered several issues running some of the tests against our Python packages. Once all tests have been patched to run against Python 3, then we should be able to use pipenv when running them against Python 2. Note that this patch allows tests to run against Python 3, but there are plenty of issues preventing them from passing. With this patch in place we can start to add Python 3 support to our packages and have the tests running in CI to ensure we don't regress back to just supporting Python 2. MozReview-Commit-ID: BuU5gZK83hL IHG: changed taskcluster/ci/source-test/python.yml
python/Pipfile
python/Pipfile.lock
python/mach/setup.py
python/mach_commands.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/virtualenv.py
python/mozversioncontrol/setup.py
new file mode 100644
--- /dev/null
+++ b/python/Pipfile
@@ -0,0 +1,36 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+"d5b4a14" = {path = "./mach"}
+"8ddb376" = {path = "./mozbuild"}
+"b3ddbcf" = {path = "./mozterm"}
+"38a4a9a" = {path = "./mozversioncontrol"}
+"26d92fb" = {path = "./../config/mozunit"}
+"cea2946" = {path = "./../testing/mozbase/manifestparser"}
+"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
+"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
+"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
+"58d0848" = {path = "./../testing/mozbase/mozfile"}
+"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
+"7329809" = {path = "./../testing/mozbase/mozinfo"}
+"501835d" = {path = "./../testing/mozbase/mozinstall"}
+"807c1c5" = {path = "./../testing/mozbase/mozlog"}
+"e09e103" = {path = "./../testing/mozbase/moznetwork"}
+"132adec" = {path = "./../testing/mozbase/mozprocess"}
+"d88f467" = {path = "./../testing/mozbase/mozprofile"}
+"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
+"6477f20" = {path = "./../testing/mozbase/moztest"}
+"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
+"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
+"110bcc4" = {path = "./../third_party/python/jsmin"}
+"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
+"c2c21d9" = {path = "./../third_party/python/py"}
+"f4b00e9" = {path = "./../third_party/python/pytest"}
+"053111f" = {path = "./../third_party/python/requests"}
+"d250320" = {path = "./../third_party/python/six"}
+"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
+
+[dev-packages]
new file mode 100644
--- /dev/null
+++ b/python/Pipfile.lock
@@ -0,0 +1,106 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "053111f": {
+            "path": "./../third_party/python/requests"
+        },
+        "110bcc4": {
+            "path": "./../third_party/python/jsmin"
+        },
+        "132adec": {
+            "path": "./../testing/mozbase/mozprocess"
+        },
+        "195ae2e": {
+            "path": "./../testing/mozbase/mozdebug"
+        },
+        "1de94f2": {
+            "path": "./../testing/mozbase/mozrunner"
+        },
+        "26d92fb": {
+            "path": "./../config/mozunit"
+        },
+        "38a4a9a": {
+            "path": "./mozversioncontrol"
+        },
+        "47200d8": {
+            "markers": "python_version < '3'",
+            "path": "./../third_party/python/futures"
+        },
+        "501835d": {
+            "path": "./../testing/mozbase/mozinstall"
+        },
+        "58d0848": {
+            "path": "./../testing/mozbase/mozfile"
+        },
+        "6477f20": {
+            "path": "./../testing/mozbase/moztest"
+        },
+        "7329809": {
+            "path": "./../testing/mozbase/mozinfo"
+        },
+        "807c1c5": {
+            "path": "./../testing/mozbase/mozlog"
+        },
+        "8dab59a": {
+            "path": "./../testing/mozbase/mozdevice"
+        },
+        "8ddb376": {
+            "path": "./mozbuild"
+        },
+        "b3ddbcf": {
+            "path": "./mozterm"
+        },
+        "c2c21d9": {
+            "path": "./../third_party/python/py"
+        },
+        "c49d32a": {
+            "markers": "python_version < '3.3'",
+            "path": "./../third_party/python/mock-1.0.0"
+        },
+        "cea2946": {
+            "path": "./../testing/mozbase/manifestparser"
+        },
+        "d250320": {
+            "path": "./../third_party/python/six"
+        },
+        "d5b4a14": {
+            "path": "./mach"
+        },
+        "d88f467": {
+            "path": "./../testing/mozbase/mozprofile"
+        },
+        "e09e103": {
+            "path": "./../testing/mozbase/moznetwork"
+        },
+        "f1d74ca": {
+            "path": "./../testing/mozbase/mozversion"
+        },
+        "f1de77a": {
+            "markers": "python_version < '3.3'",
+            "path": "./../third_party/python/which"
+        },
+        "f4b00e9": {
+            "path": "./../third_party/python/pytest"
+        },
+        "fd0b608": {
+            "path": "./../testing/mozbase/mozhttpd"
+        },
+        "ffcf6e6": {
+            "path": "./../testing/mozbase/mozcrash"
+        }
+    },
+    "develop": {}
+}
--- a/python/mach/setup.py
+++ b/python/mach/setup.py
@@ -15,17 +15,17 @@ README = open('README.rst').read()
 setup(
     name='mach',
     description='Generic command line command dispatching framework.',
     long_description=README,
     license='MPL 2.0',
     author='Gregory Szorc',
     author_email='gregory.szorc@gmail.com',
     url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach',
-    packages=['mach'],
+    packages=['mach', 'mach.mixin'],
     version=VERSION,
     classifiers=[
         'Environment :: Console',
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
         'Natural Language :: English',
     ],
     install_requires=[
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -25,16 +25,18 @@ from mozbuild.base import (
 )
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('python', category='devenv',
              description='Run Python.')
     @CommandArgument('--no-virtualenv', action='store_true',
                      help='Do not set up a virtualenv')
     @CommandArgument('args', nargs=argparse.REMAINDER)
@@ -60,16 +62,20 @@ class MachCommands(MachCommandBase):
                                 append_env=append_env)
 
     @Command('python-test', category='testing',
              description='Run Python unit tests with an appropriate test runner.')
     @CommandArgument('-v', '--verbose',
                      default=False,
                      action='store_true',
                      help='Verbose output.')
+    @CommandArgument('--three',
+                     default=False,
+                     action='store_true',
+                     help='Run tests using Python 3.')
     @CommandArgument('-j', '--jobs',
                      default=1,
                      type=int,
                      help='Number of concurrent jobs to run. Default is 1.')
     @CommandArgument('--subsuite',
                      default=None,
                      help=('Python subsuite to run. If not specified, all subsuites are run. '
                            'Use the string `default` to only run tests without a subsuite.'))
@@ -86,18 +92,23 @@ class MachCommands(MachCommandBase):
             mozfile.remove(tempdir)
 
     def run_python_tests(self,
                          tests=None,
                          test_objects=None,
                          subsuite=None,
                          verbose=False,
                          jobs=1,
+                         three=False,
                          **kwargs):
-        self._activate_virtualenv()
+        if three:
+            # use pipenv to run tests against Python 3
+            self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
+        else:
+            self._activate_virtualenv()
 
         if test_objects is None:
             from moztest.resolve import TestResolver
             resolver = self._spawn(TestResolver)
             # If we were given test paths, try to find tests matching them.
             test_objects = resolver.resolve_tests(paths=tests, flavor='python')
         else:
             # We've received test_objects from |mach test|. We need to ignore
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -752,21 +752,21 @@ class MozbuildObject(ProcessExecutionMix
         self._activate_virtualenv()
         pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
         if not os.path.exists(pipenv):
             pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
             self.virtualenv_manager.install_pip_requirements(
                 pipenv_reqs, require_hashes=False, vendored=True)
         return pipenv
 
-    def activate_pipenv(self, path):
+    def activate_pipenv(self, path, args=None):
         if not os.path.exists(path):
             raise Exception('Pipfile not found: %s.' % path)
         self.ensure_pipenv()
-        self.virtualenv_manager.activate_pipenv(path)
+        self.virtualenv_manager.activate_pipenv(path, args)
 
 
 class MachCommandBase(MozbuildObject):
     """Base class for mach command providers that wish to be MozbuildObjects.
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
     """
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -528,28 +528,29 @@ class VirtualenvManager(object):
         # This will confuse pip and cause the package to attempt to install
         # against the executing interpreter. By creating a new process, we
         # force the virtualenv's interpreter to be used and all is well.
         # It /might/ be possible to cheat and set sys.executable to
         # self.python_path. However, this seems more risk than it's worth.
         subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
             stderr=subprocess.STDOUT)
 
-    def activate_pipenv(self, pipfile):
+    def activate_pipenv(self, pipfile, args=None):
         """Install a Pipfile located at path and activate environment"""
         pipenv = os.path.join(self.bin_path, 'pipenv')
         env = os.environ.copy()
         env.update({
             'PIPENV_IGNORE_VIRTUALENVS': '1',
             'PIPENV_PIPFILE': pipfile,
             'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
         })
 
+        args = args or []
         subprocess.check_call(
-            [pipenv, 'install', '--deploy'],
+            [pipenv, 'install'] + args,
             stderr=subprocess.STDOUT,
             env=env)
 
         self.virtualenv_root = subprocess.check_output(
             [pipenv, '--venv'],
             stderr=subprocess.STDOUT,
             env=env).rstrip()
 
new file mode 100644
--- /dev/null
+++ b/python/mozversioncontrol/setup.py
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup, find_packages
+
+VERSION = '0.1'
+
+setup(
+    author='Mozilla Foundation',
+    author_email='Mozilla Release Engineering',
+    name='mozversioncontrol',
+    description='Mozilla version control functionality',
+    license='MPL 2.0',
+    packages=find_packages(),
+    version=VERSION,
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Topic :: Software Development :: Build Tools',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: CPython',
+    ],
+    keywords='mozilla',
+)