Bug 1339178 - Use pytest to run python-tests, r?davehunt draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 29 Aug 2017 14:50:33 -0400
changeset 656762 b7d72c302cc50263eef2a3afb0d20202e69c7388
parent 656761 7941be087d4d1251a90f84c0a7162cffa4b1799d
child 729224 986b90172ef57c821976c8b0c29d0b736e387ffe
push id77303
push userahalberstadt@mozilla.com
push dateThu, 31 Aug 2017 15:57:40 +0000
reviewersdavehunt
bugs1339178
milestone57.0a1
Bug 1339178 - Use pytest to run python-tests, r?davehunt This switches most tests over to use pytest as the runner instead of unittest (taking advantage of the fact that pytest can run unittest based tests). There were a couple tests that had failures when swithing to pytest: config/tests/unit-expandlibs.py xpcom/idl-parser/xpidl/runtests.py For these tests, I added a runwith='unittest' argument so that they still run the same way as before. Once we fix them to use pytest, the unittest logic in mozunit.py can be deleted. MozReview-Commit-ID: Gcsz6z8MeOi
config/mozunit.py
config/tests/unit-expandlibs.py
python/mach_commands.py
python/mozbuild/mozbuild/test/test_pythonutil.py
python/mozlint/test/test_cli.py
python/mozlint/test/test_formatters.py
python/mozlint/test/test_parser.py
python/mozlint/test/test_roller.py
python/mozlint/test/test_types.py
python/mozlint/test/test_vcs.py
testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
testing/mochitest/tests/python/test_basic_mochitest_plain.py
testing/mochitest/tests/python/test_get_active_tests.py
testing/mozbase/mozcrash/tests/test.py
testing/mozbase/mozlog/tests/test_logger.py
xpcom/idl-parser/xpidl/runtests.py
--- a/config/mozunit.py
+++ b/config/mozunit.py
@@ -4,16 +4,18 @@
 
 import inspect
 import os
 import sys
 import unittest
 from StringIO import StringIO
 from unittest import TextTestRunner as _TestRunner, TestResult as _TestResult
 
+import pytest
+
 '''Helper to make python unit tests report the way that the Mozilla
 unit test infrastructure expects tests to report.
 
 Usage:
 
 import mozunit
 
 if __name__ == '__main__':
@@ -204,9 +206,24 @@ class MockedOpen(object):
         abspath = normcase(os.path.abspath(p) + os.sep)
         if any(f.startswith(abspath) for f in self.files):
             return True
 
         return self._orig_path_exists(p)
 
 
 def main(*args, **kwargs):
-    unittest.main(testRunner=MozTestRunner(), *args, **kwargs)
+    runwith = kwargs.pop('runwith', 'pytest')
+
+    if runwith == 'unittest':
+        unittest.main(testRunner=MozTestRunner(), *args, **kwargs)
+    else:
+        args = list(args)
+        if os.environ.get('MACH_STDOUT_ISATTY') and not any(a.startswith('--color') for a in args):
+            args.append('--color=yes')
+
+        module = __import__('__main__')
+        args.extend([
+            '--verbose',
+            '-p', 'mozlog.pytest_mozlog.plugin',
+            module.__file__,
+        ])
+        sys.exit(pytest.main(args))
--- a/config/tests/unit-expandlibs.py
+++ b/config/tests/unit-expandlibs.py
@@ -423,9 +423,9 @@ class TestSymbolOrder(unittest.TestCase)
         config.LD_PRINT_ICF_SECTIONS = '-Wl,--print-icf-sections'
         args = ExpandArgsMore(['foo', '-bar', 'bar.o', 'foo.o'])
         self.assertEqual(args._getOrderedSections(['hello', '_Z6barbazv']), ['.text.hello', '.text.hi', '.text.hot._Z6barbazv'])
         self.assertEqual(args._getOrderedSections(['_ZThn4_6foobarv', 'hi', '_Z6barbazv']), ['.text._Z6foobarv', '.text._ZThn4_6foobarv', '.text.hi', '.text.hello', '.text.hot._Z6barbazv'])
         subprocess.Popen = subprocess_popen
 
 
 if __name__ == '__main__':
-    mozunit.main()
+    mozunit.main(runwith='unittest')
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -201,17 +201,17 @@ class MachCommands(MachCommandBase):
         def _line_handler(line):
             if not file_displayed_test:
                 output = ('Ran' in line or 'collected' in line or
                           line.startswith('TEST-'))
                 if output:
                     file_displayed_test.append(True)
 
             # Hack to make sure treeherder highlights pytest failures
-            if line.endswith('FAILED'):
+            if 'FAILED' in line.rsplit(' ', 1)[-1]:
                 line = line.replace('FAILED', 'TEST-UNEXPECTED-FAIL')
 
             _log(line)
 
         _log(test_path)
         cmd = [self.virtualenv_manager.python_path, test_path]
         env = os.environ.copy()
         env[b'PYTHONDONTWRITEBYTECODE'] = b'1'
--- a/python/mozbuild/mozbuild/test/test_pythonutil.py
+++ b/python/mozbuild/mozbuild/test/test_pythonutil.py
@@ -7,17 +7,18 @@ from mozunit import main
 import os
 import unittest
 
 
 class TestIterModules(unittest.TestCase):
     def test_iter_modules_in_path(self):
         mozbuild_path = os.path.normcase(os.path.dirname(os.path.dirname(__file__)))
         paths = list(iter_modules_in_path(mozbuild_path))
-        self.assertEquals(sorted(paths), [
+        self.assertEquals(set(paths), set([
             os.path.join(os.path.abspath(mozbuild_path), '__init__.py'),
             os.path.join(os.path.abspath(mozbuild_path), 'pythonutil.py'),
+            os.path.join(os.path.abspath(mozbuild_path), 'test', '__init__.py'),
             os.path.join(os.path.abspath(mozbuild_path), 'test', 'test_pythonutil.py'),
-        ])
+        ]))
 
 
 if __name__ == '__main__':
     main()
--- a/python/mozlint/test/test_cli.py
+++ b/python/mozlint/test/test_cli.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 from distutils.spawn import find_executable
 
+import mozunit
 import pytest
 
 from mozlint import cli
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 @pytest.fixture
@@ -53,9 +53,9 @@ def test_cli_run_with_edit(run, parser, 
     assert "foobar.js: line 2, col 1, Error" in out[2]
 
     del os.environ['EDITOR']
     with pytest.raises(SystemExit):
         parser.parse_args(['--edit'])
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_formatters.py
+++ b/python/mozlint/test/test_formatters.py
@@ -1,18 +1,18 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import json
-import sys
 from collections import defaultdict
 
+import mozunit
 import pytest
 
 from mozlint import ResultContainer
 from mozlint import formatters
 
 
 EXPECTED = {
     'compact': {
@@ -99,9 +99,9 @@ def test_json_formatter(results):
 
     slots = ResultContainer.__slots__
     for errors in formatted.values():
         for err in errors:
             assert all(s in err for s in slots)
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_parser.py
+++ b/python/mozlint/test/test_parser.py
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.parser import Parser
 from mozlint.errors import (
     LinterNotFound,
     LinterParseError,
 )
 
@@ -53,9 +53,9 @@ def test_parse_invalid_linter(parse, lin
 
 
 def test_parse_non_existent_linter(parse):
     with pytest.raises(LinterNotFound):
         parse('missing_file.lint')
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import sys
 
+import mozunit
 import pytest
 
 from mozlint import ResultContainer
 from mozlint.errors import LintersNotConfigured, LintException
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -74,9 +75,9 @@ def test_roll_with_failure_code(lint, li
 
     assert lint.failed is None
     result = lint.roll(files)
     assert len(result) == 0
     assert lint.failed == ['BadReturnCodeLinter']
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_types.py
+++ b/python/mozlint/test/test_types.py
@@ -1,15 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.result import ResultContainer
 
 
 @pytest.fixture
 def path(filedir):
     def _path(name):
@@ -46,9 +46,9 @@ def test_no_filter(lint, lintdir, files)
     assert len(result) == 0
 
     lint.lintargs['use_filters'] = False
     result = lint.roll(files)
     assert len(result) == 2
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/python/mozlint/test/test_vcs.py
+++ b/python/mozlint/test/test_vcs.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 import subprocess
-import sys
 
+import mozunit
 import pytest
 
 from mozlint.vcs import VCSHelper, vcs_class
 
 
 setup = {
     'hg': [
         """
@@ -114,9 +114,9 @@ def test_vcs_helper(repo):
 
     assert_files(vcs.by_workdir('all'), [])
     assert_files(vcs.by_workdir('staged'), [])
     assert_files(vcs.by_outgoing(), ['bar', 'baz'])
     assert_files(vcs.by_outgoing(remotepath), ['bar', 'baz'])
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/pytest.ini
@@ -1,7 +1,2 @@
 [pytest]
-# Early-load pytest_mozlog plugin to replace terminal reporter.
-# Adding pytest_mozlog plugin to conftest.py registers the plugin
-# too late for tests to recognize mozlog options.
-# This manual registration of plugin is needed for running these
-# tests in mach, whose virtualenv setup does not call mozlog's setup.py
-addopts = -p mozlog.pytest_mozlog.plugin -p no:terminalreporter
+addopts = -p no:terminalreporter
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import types
 import urllib2
 
+import mozunit
 import pytest
 
 from wptserve.handlers import json_handler
 
 from marionette_harness.runner import httpd
 
 here = os.path.abspath(os.path.dirname(__file__))
 parent = os.path.dirname(here)
@@ -81,11 +82,9 @@ def test_handler(server):
 
     url = server.get_url("/httpd/test_handler")
     body = urllib2.urlopen(url).read()
     res = json.loads(body)
     assert res["count"] == counter
 
 
 if __name__ == "__main__":
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
@@ -1,11 +1,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import mozunit
 import pytest
 
 from marionette_harness.runtests import MarionetteArguments
 
 
 @pytest.mark.parametrize("socket_timeout", ['A', '10', '1B-', '1C2', '44.35'])
 def test_parse_arg_socket_timeout(socket_timeout):
     argv = ['marionette', '--socket-timeout', socket_timeout]
@@ -23,11 +24,9 @@ def test_parse_arg_socket_timeout(socket
             parser.parse_args(args=argv)
         assert ex.value.code == 2
     else:
         args = parser.parse_args(args=argv)
         assert hasattr(args, 'socket_timeout') and args.socket_timeout == float(socket_timeout)
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import mozunit
 import pytest
 
 from mock import Mock, patch, sentinel
 
 import marionette_harness.marionette_test as marionette_test
 
 from marionette_harness.runtests import MarionetteTestRunner, MarionetteHarness, cli
 
@@ -99,11 +100,9 @@ def test_harness_sets_up_default_test_ha
     """
     harness = MarionetteHarness(args=mach_parsed_kwargs)
     mach_parsed_kwargs.pop('tests')
     runner = harness._runner_class(**mach_parsed_kwargs)
     assert marionette_test.MarionetteTestCase in runner.test_handlers
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -1,13 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import manifestparser
+import mozunit
 import pytest
 
 from mock import Mock, patch, mock_open, sentinel, DEFAULT
 
 from marionette_harness.runtests import MarionetteTestRunner
 
 
 @pytest.fixture
@@ -502,11 +503,9 @@ def test_option_run_until_failure(mach_p
         assert runner.run_until_failure == run_until_failure
         if repeat is None:
             assert runner.repeat == 30
         else:
             assert runner.repeat == repeat
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import mozunit
 import pytest
 
 from marionette_harness import MarionetteTestResult
 
 
 @pytest.fixture
 def empty_marionette_testcase():
     """ Testable MarionetteTestCase class """
@@ -45,11 +46,9 @@ def test_crash_is_recorded_as_error(empt
     assert result.shouldStop == has_crashed
     if has_crashed:
         assert len(result.errors) == 1
     else:
         assert len(result.errors) == 0
 
 
 if __name__ == '__main__':
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import types
 
+import mozunit
 import pytest
 
 from marionette_harness.runner import serve
 from marionette_harness.runner.serve import iter_proc, iter_url
 
 
 def teardown_function(func):
     for server in [server for server in iter_proc(serve.servers) if server.is_alive]:
@@ -58,11 +59,9 @@ def test_iter_url():
 
 def test_where_is():
     serve.start()
     assert serve.where_is("/") == serve.servers["http"][1].get_url("/")
     assert serve.where_is("/", on="https") == serve.servers["https"][1].get_url("/")
 
 
 if __name__ == "__main__":
-    import sys
-    sys.exit(pytest.main(
-        ['--log-tbpl=-', __file__]))
+    mozunit.main('--log-tbpl=-')
--- a/testing/mochitest/tests/python/test_basic_mochitest_plain.py
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import sys
 
+import mozunit
 import pytest
 
 from conftest import build, filter_action
 
 sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
 from mozharness.base.log import INFO, WARNING, ERROR
 from mozharness.base.errors import BaseErrorList
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
@@ -157,9 +158,9 @@ def test_output_leak(monkeypatch, runtes
 
     errors = filter_action('log', lines)
     errors = [e for e in errors if e['level'] == 'ERROR']
     assert len(errors) == 1
     assert 'leakcheck' in errors[0]['message']
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/mochitest/tests/python/test_get_active_tests.py
+++ b/testing/mochitest/tests/python/test_get_active_tests.py
@@ -1,20 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import os
-import sys
 from argparse import Namespace
 
 from manifestparser import TestManifest
 
+import mozunit
 import pytest
 
 
 @pytest.fixture
 def get_active_tests(setup_harness_root, parser):
     runtests = pytest.importorskip('runtests')
     md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
 
@@ -83,9 +83,9 @@ prefs=
 prefs=foo=bar
 [files/test_fail.html]
 """)[1]
     with pytest.raises(SystemExit):
         get_active_tests(**options)
 
 
 if __name__ == '__main__':
-    sys.exit(pytest.main(['--verbose', __file__]))
+    mozunit.main()
--- a/testing/mozbase/mozcrash/tests/test.py
+++ b/testing/mozbase/mozcrash/tests/test.py
@@ -15,17 +15,20 @@ import StringIO
 
 import mozunit
 
 import mozcrash
 import mozhttpd
 import mozlog.unstructured as mozlog
 
 # Make logs go away
-log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull))
+try:
+    log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull))
+except ValueError:
+    pass
 
 
 def popen_factory(stdouts):
     """
     Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that
     should return an iterable for the stdout of each process in turn.
     """
     class mock_popen(object):
--- a/testing/mozbase/mozlog/tests/test_logger.py
+++ b/testing/mozbase/mozlog/tests/test_logger.py
@@ -228,17 +228,17 @@ class Loggable(mozlog.LoggingMixin):
 class TestLoggingMixin(unittest.TestCase):
     """Tests basic use of LoggingMixin"""
 
     def test_mixin(self):
         loggable = Loggable()
         self.assertTrue(not hasattr(loggable, "_logger"))
         loggable.log(mozlog.INFO, "This will instantiate the logger")
         self.assertTrue(hasattr(loggable, "_logger"))
-        self.assertEqual(loggable._logger.name, "__main__.Loggable")
+        self.assertEqual(loggable._logger.name, "test_logger.Loggable")
 
         self.assertRaises(ValueError, loggable.set_logger,
                           "not a logger")
 
         logger = mozlog.MozLogger('test.mixin')
         handler = ListHandler()
         logger.addHandler(handler)
         loggable.set_logger(logger)
--- a/xpcom/idl-parser/xpidl/runtests.py
+++ b/xpcom/idl-parser/xpidl/runtests.py
@@ -105,9 +105,9 @@ void getBar();
             def write(self, s):
                 pass
         try:
             header.print_header(i, FdMock(), filename='f')
         except Exception as e:
             self.assertEqual(e.args[0], "Unexpected overloaded virtual method GetBar in interface foo")
 
 if __name__ == '__main__':
-    mozunit.main()
+    mozunit.main(runwith='unittest')