Bug 1048446 - [python-test] Create a mochitest selftest harness, r?jmaher
This will create a mochitest selftest harness based on |mach python-test|. There
is also a basic test that checks whether TEST-PASS and TEST-UNEXPECTED-FAIL work.
MozReview-Commit-ID: Jqyhbj7nC6z
--- a/moz.build
+++ b/moz.build
@@ -64,16 +64,17 @@ DIRS += [
'testing/mozbase',
'third_party/python',
]
if not CONFIG['JS_STANDALONE']:
# These python manifests are included here so they get picked up without an objdir
PYTHON_UNITTEST_MANIFESTS += [
'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
+ 'testing/mochitest/tests/python/python.ini',
]
CONFIGURE_SUBST_FILES += [
'tools/update-packaging/Makefile',
]
CONFIGURE_DEFINE_FILES += [
'mozilla-config.h',
]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/conftest.py
@@ -0,0 +1,178 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import shutil
+import sys
+from argparse import Namespace
+from cStringIO import StringIO
+
+import pytest
+import requests
+
+import mozfile
+import mozinstall
+from manifestparser import TestManifest
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def filter_action(action, lines):
+ return filter(lambda x: x['action'] == action, lines)
+
+
+def _get_harness_root():
+ # Check if there is a local build
+ harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
+ if os.path.isdir(harness_root):
+ return harness_root
+
+ # Check if it was previously set up by another test
+ harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', 'mochitest')
+ if os.path.isdir(harness_root):
+ return harness_root
+
+ # Check if there is a test package to download
+ if 'GECKO_INSTALLER_URL' in os.environ:
+ base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+ test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+ dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+ for name in test_packages['mochitest']:
+ url = base_url + '/' + name
+ bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+ r = requests.get(url, stream=True)
+ with open(bundle, 'w+b') as fh:
+ for chunk in r.iter_content(chunk_size=1024):
+ fh.write(chunk)
+
+ mozfile.extract(bundle, dest)
+
+ return os.path.join(dest, 'mochitest')
+
+ # Couldn't find a harness root, let caller do error handling.
+ return None
+
+
+@pytest.fixture(scope='session')
+def setup_harness_root():
+ harness_root = _get_harness_root()
+ if harness_root:
+ sys.path.insert(0, harness_root)
+
+ # Link the test files to the test package so updates are automatically
+ # picked up. Fallback to copy on Windows.
+ test_root = os.path.join(harness_root, 'tests', 'selftests')
+ if not os.path.exists(test_root):
+ files = os.path.join(here, 'files')
+ if hasattr(os, 'symlink'):
+ os.symlink(files, test_root)
+ else:
+ shutil.copytree(files, test_root)
+
+ elif 'GECKO_INSTALLER_URL' in os.environ:
+ # The mochitest tests will run regardless of whether a build exists or not.
+ # In a local environment, they should simply be skipped if setup fails. But
+ # in automation, we'll need to make sure an error is propagated up.
+ pytest.fail(HARNESS_ROOT_NOT_FOUND)
+ else:
+ # Tests will be marked skipped by the calls to pytest.importorskip() below.
+ # We are purposefully not failing here because running |mach python-test|
+ # without a build is a perfectly valid use case.
+ pass
+
+
+@pytest.fixture(scope='session')
+def binary():
+ try:
+ return build.get_binary_path()
+ except:
+ pass
+
+ app = 'firefox'
+ bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
+ if os.path.isdir(bindir):
+ try:
+ return mozinstall.get_binary(bindir, app_name=app)
+ except:
+ pass
+
+ if 'GECKO_INSTALLER_URL' in os.environ:
+ bindir = mozinstall.install(
+ os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
+ return mozinstall.get_binary(bindir, app_name='firefox')
+
+
+@pytest.fixture(scope='function')
+def parser(request):
+ parser = pytest.importorskip('mochitest_options')
+
+ app = getattr(request.module, 'APP', 'generic')
+ return parser.MochitestArgumentParser(app=app)
+
+
+@pytest.fixture(scope='function')
+def runtests(setup_harness_root, binary, parser, request):
+ """Creates an easy to use entry point into the mochitest harness.
+
+ :returns: A function with the signature `*tests, **opts`. Each test is a file name
+ (relative to the `files` dir). At least one is required. The opts are
+ used to override the default mochitest options, they are optional.
+ """
+ runtests = pytest.importorskip('runtests')
+
+ mochitest_root = runtests.SCRIPT_DIR
+ test_root = os.path.join(mochitest_root, 'tests', 'selftests')
+
+ buf = StringIO()
+ options = vars(parser.parse_args([]))
+ options.update({
+ 'app': binary,
+ 'keep_open': False,
+ 'log_raw': [buf],
+ })
+
+ if not os.path.isdir(runtests.build_obj.bindir):
+ package_root = os.path.dirname(mochitest_root)
+ options.update({
+ 'certPath': os.path.join(package_root, 'certs'),
+ 'utilityPath': os.path.join(package_root, 'bin'),
+ })
+ options['extraProfileFiles'].append(os.path.join(package_root, 'bin', 'plugins'))
+
+ options.update(getattr(request.module, 'OPTIONS', {}))
+
+ def normalize(test):
+ return {
+ 'name': test,
+ 'relpath': test,
+ 'path': os.path.join(test_root, test),
+ # add a dummy manifest file because mochitest expects it
+ 'manifest': os.path.join(test_root, 'mochitest.ini'),
+ }
+
+ def inner(*tests, **opts):
+ assert len(tests) > 0
+
+ manifest = TestManifest()
+ manifest.tests.extend(map(normalize, tests))
+ options['manifestFile'] = manifest
+ options.update(opts)
+
+ result = runtests.run_test_harness(parser, Namespace(**options))
+ out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
+ buf.close()
+ return result, out
+ return inner
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_fail.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+ <meta charset="utf-8">
+ <title>Test Fail</title>
+ <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+ <script type="application/javascript">
+ ok(false, "Test is ok");
+ </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/files/test_pass.html
@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+ <meta charset="utf-8">
+ <title>Test Pass</title>
+ <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+ <script type="application/javascript">
+ ok(true, "Test is ok");
+ </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/python.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+subsuite = mochitest
+sequential = true
+
+[test_basic_mochitest_plain.py]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import sys
+
+import pytest
+
+from conftest import build, filter_action
+
+sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
+from mozharness.base.log import INFO, WARNING
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
+
+def get_mozharness_status(lines, status):
+ parser = StructuredOutputParser(
+ config={'log_level': INFO},
+ error_list=BaseErrorList+HarnessErrorList,
+ strict=False,
+ suite_category='mochitest',
+ )
+
+ for line in lines:
+ parser.parse_single_line(json.dumps(line))
+ return parser.evaluate_parser(status)
+
+
+def test_output_pass(runtests):
+ status, lines = runtests('test_pass.html')
+ assert status == 0
+
+ tbpl_status, log_level = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_SUCCESS
+ assert log_level == WARNING
+
+ lines = filter_action('test_status', lines)
+ assert len(lines) == 1
+ assert lines[0]['status'] == 'PASS'
+
+
+def test_output_fail(runtests):
+ from runtests import build_obj
+
+ status, lines = runtests('test_fail.html')
+ assert status == 1
+
+ tbpl_status, log_level = get_mozharness_status(lines, status)
+ assert tbpl_status == TBPL_WARNING
+ assert log_level == WARNING
+
+ lines = filter_action('test_status', lines)
+
+ # If we are running with a build_obj, the failed status will be
+ # logged a second time at the end of the run.
+ if build_obj:
+ assert len(lines) == 2
+ else:
+ assert len(lines) == 1
+ assert lines[0]['status'] == 'FAIL'
+
+ if build_obj:
+ assert set(lines[0].keys()) == set(lines[1].keys())
+ assert set(lines[0].values()) == set(lines[1].values())
+
+
+if __name__ == '__main__':
+ sys.exit(pytest.main(['--verbose', __file__]))