Bug 1227367 - Make harness tests compatible with mach python-test; r=automatedtester; r?gps draft
authorMaja Frydrychowicz <mjzffr@gmail.com>
Thu, 31 Mar 2016 01:16:11 -0400
changeset 346161 88b3d2c08bf307f5dc1c45a88ca38d84c3b9d170
parent 346160 030c878ff28e1dcc10ee6d34c9412fdcbbd892ce
child 346162 59aa2b3b4dc10ce38a97bc46a2ea4c6a1d341af7
push id14264
push usermjzffr@gmail.com
push dateThu, 31 Mar 2016 05:20:28 +0000
reviewersautomatedtester, gps
bugs1227367
milestone48.0a1
Bug 1227367 - Make harness tests compatible with mach python-test; r=automatedtester; r?gps This uses the same approach as with mozunit to select pytest as the test runner. Add Marionette harness tests to PYTHON_UNIT_TESTS I also create a pytest plugin that modifies the pytest output to include a string expected by the ./mach python-test command. I wrote it as a named plugin, rather than just including it in conftest.py, so that it can be imported into other test suites in the future. MozReview-Commit-ID: HZWSpyjHKSr
python/mach_commands.py
testing/marionette/harness/marionette/tests/harness_unit/conftest.py
testing/marionette/harness/marionette/tests/harness_unit/mozunit_report.py
testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
testing/marionette/moz.build
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -63,17 +63,17 @@ class MachCommands(MachCommandBase):
 
         return self.run_process([self.virtualenv_manager.python_path] + args,
             pass_thru=True,  # Allow user to run Python interactively.
             ensure_exit_code=False,  # Don't throw on non-zero exit code.
             # Note: subprocess requires native strings in os.environ on Windows
             append_env={b'PYTHONDONTWRITEBYTECODE': str('1')})
 
     @Command('python-test', category='testing',
-        description='Run Python unit tests.')
+        description='Run Python unit tests with an appropriate test runner.')
     @CommandArgument('--verbose',
         default=False,
         action='store_true',
         help='Verbose output.')
     @CommandArgument('--stop',
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
@@ -82,22 +82,26 @@ class MachCommands(MachCommandBase):
         help='Tests to run. Each test can be a single file or a directory.')
     def python_test(self,
                     tests=[],
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
                     stop=False):
         self._activate_virtualenv()
+        # Some tests may require pytest
+        self.virtualenv_manager.install_pip_package('pytest==2.9.1')
 
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
         # is a simple way to keep environments separate, at the price of
-        # launching Python multiple times. This also runs tests via mozunit,
+        # launching Python multiple times. Most tests are run via mozunit,
         # which produces output in the format Mozilla infrastructure expects.
+        # Some tests are run via pytest, and these should be equipped with a
+        # local mozunit_report plugin to meet output expectations.
         return_code = 0
         if test_objects is None:
             # If we're not being called from `mach test`, do our own
             # test resolution.
             from mozbuild.testing import TestResolver
             resolver = self._spawn(TestResolver)
             if tests:
                 # If we were given test paths, try to find tests matching them.
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette/tests/harness_unit/conftest.py
@@ -0,0 +1,5 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+pytest_plugins = "mozunit_report"
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette/tests/harness_unit/mozunit_report.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+
+def pytest_configure(config):
+    config.option.verbose -= config.option.quiet
+    reporter = MozunitReport(config, sys.stdout)
+    config.pluginmanager.register(reporter, 'mozunit_reporter')
+
+
+class MozunitReport(object):
+    """
+    Modifies pytest terminal output to meet expectations of Mozilla
+    automation
+    """
+    def __init__(self, config, file=None):
+        import _pytest.config
+        if file is None:
+            file = sys.stdout
+        self._tw = _pytest.config.create_terminal_writer(config, file)
+
+    def pytest_collection_modifyitems(self):
+        # mach python-test expects at least one line that starts with
+        # 'TEST-'
+        self._tw.write('TEST-SUITE-START')
+
--- a/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
@@ -25,8 +25,12 @@ def harness_class(request):
 @pytest.mark.parametrize(
     "num_failures,exit_code",
     [(0, 0), (1, 10), (None, 1)],
 )
 def test_cli_exit_code(num_failures, exit_code, harness_class):
     with pytest.raises(SystemExit) as err:
         runtests.cli(harness_class=harness_class)
     assert err.value.code == exit_code
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/moz.build
+++ b/testing/marionette/moz.build
@@ -4,11 +4,14 @@
 
 DIRS += ["components"]
 
 JAR_MANIFESTS += ["jar.mn"]
 MARIONETTE_UNIT_MANIFESTS += ["harness/marionette/tests/unit/unit-tests.ini"]
 MARIONETTE_UPDATE_MANIFESTS += ["harness/marionette/tests/update-tests.ini"]
 MARIONETTE_WEBAPI_MANIFESTS += ["harness/marionette/tests/webapi-tests.ini"]
 XPCSHELL_TESTS_MANIFESTS += ["unit.ini"]
+PYTHON_UNIT_TESTS += [
+    "harness/marionette/tests/harness_unit/test_marionette_runner.py",
+]
 
 with Files("**"):
     BUG_COMPONENT = ("Testing", "Marionette")