Bug 1227367 - Make harness tests compatible with mach python-test; r=automatedtester; r?gps
This uses the same approach as with mozunit to select pytest as the test runner.
Add Marionette harness tests to PYTHON_UNIT_TESTS
I also create a pytest plugin that modifies the pytest output to include
a string expected by the ./mach python-test command. I wrote it as a named
plugin, rather than just including it in conftest.py, so that it can be
imported into other test suites in the future.
MozReview-Commit-ID: HZWSpyjHKSr
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -63,17 +63,17 @@ class MachCommands(MachCommandBase):
return self.run_process([self.virtualenv_manager.python_path] + args,
pass_thru=True, # Allow user to run Python interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code.
# Note: subprocess requires native strings in os.environ on Windows
append_env={b'PYTHONDONTWRITEBYTECODE': str('1')})
@Command('python-test', category='testing',
- description='Run Python unit tests.')
+ description='Run Python unit tests with an appropriate test runner.')
@CommandArgument('--verbose',
default=False,
action='store_true',
help='Verbose output.')
@CommandArgument('--stop',
default=False,
action='store_true',
help='Stop running tests after the first error or failure.')
@@ -82,22 +82,26 @@ class MachCommands(MachCommandBase):
help='Tests to run. Each test can be a single file or a directory.')
def python_test(self,
tests=[],
test_objects=None,
subsuite=None,
verbose=False,
stop=False):
self._activate_virtualenv()
+ # Some tests may require pytest
+ self.virtualenv_manager.install_pip_package('pytest==2.9.1')
# Python's unittest, and in particular discover, has problems with
# clashing namespaces when importing multiple test modules. What follows
# is a simple way to keep environments separate, at the price of
- # launching Python multiple times. This also runs tests via mozunit,
+ # launching Python multiple times. Most tests are run via mozunit,
# which produces output in the format Mozilla infrastructure expects.
+ # Some tests are run via pytest, and these should be equipped with a
+ # local mozunit_report plugin to meet output expectations.
return_code = 0
if test_objects is None:
# If we're not being called from `mach test`, do our own
# test resolution.
from mozbuild.testing import TestResolver
resolver = self._spawn(TestResolver)
if tests:
# If we were given test paths, try to find tests matching them.
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette/tests/harness_unit/conftest.py
@@ -0,0 +1,5 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+pytest_plugins = "mozunit_report"
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette/tests/harness_unit/mozunit_report.py
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+
+def pytest_configure(config):
+ config.option.verbose -= config.option.quiet
+ reporter = MozunitReport(config, sys.stdout)
+ config.pluginmanager.register(reporter, 'mozunit_reporter')
+
+
+class MozunitReport(object):
+ """
+ Modifies pytest terminal output to meet expectations of Mozilla
+ automation
+ """
+ def __init__(self, config, file=None):
+ import _pytest.config
+ if file is None:
+ file = sys.stdout
+ self._tw = _pytest.config.create_terminal_writer(config, file)
+
+ def pytest_collection_modifyitems(self):
+ # mach python-test expects at least one line that starts with
+ # 'TEST-'
+ self._tw.write('TEST-SUITE-START')
+
--- a/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette/tests/harness_unit/test_marionette_runner.py
@@ -25,8 +25,12 @@ def harness_class(request):
@pytest.mark.parametrize(
"num_failures,exit_code",
[(0, 0), (1, 10), (None, 1)],
)
def test_cli_exit_code(num_failures, exit_code, harness_class):
with pytest.raises(SystemExit) as err:
runtests.cli(harness_class=harness_class)
assert err.value.code == exit_code
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(pytest.main(['--verbose', __file__]))
--- a/testing/marionette/moz.build
+++ b/testing/marionette/moz.build
@@ -4,11 +4,14 @@
DIRS += ["components"]
JAR_MANIFESTS += ["jar.mn"]
MARIONETTE_UNIT_MANIFESTS += ["harness/marionette/tests/unit/unit-tests.ini"]
MARIONETTE_UPDATE_MANIFESTS += ["harness/marionette/tests/update-tests.ini"]
MARIONETTE_WEBAPI_MANIFESTS += ["harness/marionette/tests/webapi-tests.ini"]
XPCSHELL_TESTS_MANIFESTS += ["unit.ini"]
+PYTHON_UNIT_TESTS += [
+ "harness/marionette/tests/harness_unit/test_marionette_runner.py",
+]
with Files("**"):
BUG_COMPONENT = ("Testing", "Marionette")