Bug 1392390 - Refactor common code out of mochitest selftests and into a new moztest.selftest module, r?jmaher
This includes code for downloading a Firefox binary, downloading + setting up a tests.zip and
running output through mozharness' output parsers. This is all stuff that will also be required
for the reftest selftests.
I couldn't think of a better location to put this stuff, suggestions welcome.
MozReview-Commit-ID: 59TSbsugT5T
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -70,16 +70,17 @@ mochitest-harness:
start_xvfb '1600x1200x24' 0 &&
cd /builds/worker/checkouts/gecko &&
./mach python-test --subsuite mochitest
when:
files-changed:
- 'config/mozunit.py'
- 'python/mach_commands.py'
- 'testing/mochitest/**'
+ - 'testing/mozbase/moztest/moztest/selftest/**'
- 'testing/mozharness/mozharness/base/log.py'
- 'testing/mozharness/mozharness/mozilla/structuredlog.py'
- 'testing/mozharness/mozharness/mozilla/testing/errors.py'
- 'testing/profiles/prefs_general.js'
mozbase:
description: testing/mozbase unit tests
platform:
--- a/testing/mochitest/tests/python/conftest.py
+++ b/testing/mochitest/tests/python/conftest.py
@@ -1,144 +1,45 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
-
import json
import os
-import shutil
-import sys
from argparse import Namespace
from cStringIO import StringIO
import pytest
-import requests
-import mozfile
import mozinfo
-import mozinstall
from manifestparser import TestManifest, expression
-from mozbuild.base import MozbuildObject
+from moztest.selftest.fixtures import binary, setup_test_harness # noqa
here = os.path.abspath(os.path.dirname(__file__))
-build = MozbuildObject.from_environment(cwd=here)
-
-HARNESS_ROOT_NOT_FOUND = """
-Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
-environment variable is required.
-""".lstrip()
-
-
-def filter_action(actions, lines):
- if isinstance(actions, basestring):
- actions = (actions,)
- return filter(lambda x: x['action'] in actions, lines)
-
-
-def _get_harness_root():
- # Check if there is a local build
- harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
- if os.path.isdir(harness_root):
- return harness_root
-
- # Check if it was previously set up by another test
- harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', 'mochitest')
- if os.path.isdir(harness_root):
- return harness_root
-
- # Check if there is a test package to download
- if 'GECKO_INSTALLER_URL' in os.environ:
- base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
- test_packages = requests.get(base_url + '/target.test_packages.json').json()
-
- dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
- for name in test_packages['mochitest']:
- url = base_url + '/' + name
- bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
-
- r = requests.get(url, stream=True)
- with open(bundle, 'w+b') as fh:
- for chunk in r.iter_content(chunk_size=1024):
- fh.write(chunk)
-
- mozfile.extract(bundle, dest)
-
- return os.path.join(dest, 'mochitest')
-
- # Couldn't find a harness root, let caller do error handling.
- return None
-
-
-@pytest.fixture(scope='session')
-def setup_harness_root():
- harness_root = _get_harness_root()
- if harness_root:
- sys.path.insert(0, harness_root)
-
- # Link the test files to the test package so updates are automatically
- # picked up. Fallback to copy on Windows.
- test_root = os.path.join(harness_root, 'tests', 'selftests')
- if not os.path.exists(test_root):
- files = os.path.join(here, 'files')
- if hasattr(os, 'symlink'):
- os.symlink(files, test_root)
- else:
- shutil.copytree(files, test_root)
-
- elif 'GECKO_INSTALLER_URL' in os.environ:
- # The mochitest tests will run regardless of whether a build exists or not.
- # In a local environment, they should simply be skipped if setup fails. But
- # in automation, we'll need to make sure an error is propagated up.
- pytest.fail(HARNESS_ROOT_NOT_FOUND)
- else:
- # Tests will be marked skipped by the calls to pytest.importorskip() below.
- # We are purposefully not failing here because running |mach python-test|
- # without a build is a perfectly valid use case.
- pass
-
-
-@pytest.fixture(scope='session')
-def binary():
- try:
- return build.get_binary_path()
- except:
- pass
-
- app = 'firefox'
- bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
- if os.path.isdir(bindir):
- try:
- return mozinstall.get_binary(bindir, app_name=app)
- except:
- pass
-
- if 'GECKO_INSTALLER_URL' in os.environ:
- bindir = mozinstall.install(
- os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
- return mozinstall.get_binary(bindir, app_name='firefox')
+setup_args = [os.path.join(here, 'files'), 'mochitest', 'testing/mochitest']
@pytest.fixture(scope='function')
def parser(request):
parser = pytest.importorskip('mochitest_options')
app = getattr(request.module, 'APP', 'generic')
return parser.MochitestArgumentParser(app=app)
-@pytest.fixture(scope='function')
-def runtests(setup_harness_root, binary, parser, request):
+@pytest.fixture(scope='function') # noqa: F811
+def runtests(setup_test_harness, binary, parser, request):
"""Creates an easy to use entry point into the mochitest harness.
:returns: A function with the signature `*tests, **opts`. Each test is a file name
(relative to the `files` dir). At least one is required. The opts are
used to override the default mochitest options, they are optional.
"""
+ setup_test_harness(*setup_args)
runtests = pytest.importorskip('runtests')
mochitest_root = runtests.SCRIPT_DIR
test_root = os.path.join(mochitest_root, 'tests', 'selftests')
buf = StringIO()
options = vars(parser.parse_args([]))
options.update({
@@ -176,32 +77,34 @@ def runtests(setup_harness_root, binary,
result = runtests.run_test_harness(parser, Namespace(**options))
out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
buf.close()
return result, out
return inner
-@pytest.fixture
-def build_obj(setup_harness_root):
+@pytest.fixture # noqa: F811
+def build_obj(setup_test_harness):
+ setup_test_harness(*setup_args)
mochitest_options = pytest.importorskip('mochitest_options')
return mochitest_options.build_obj
-@pytest.fixture(autouse=True)
-def skip_using_mozinfo(request, setup_harness_root):
+@pytest.fixture(autouse=True) # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
"""Gives tests the ability to skip based on values from mozinfo.
Example:
@pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
def test_foo():
pass
"""
+ setup_test_harness(*setup_args)
runtests = pytest.importorskip('runtests')
runtests.update_mozinfo()
skip_mozinfo = request.node.get_marker('skip_mozinfo')
if skip_mozinfo:
value = skip_mozinfo.args[0]
if expression.parse(value, **mozinfo.info):
pytest.skip("skipped due to mozinfo match: \n{}".format(value))
--- a/testing/mochitest/tests/python/test_basic_mochitest_plain.py
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -1,49 +1,25 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import json
import os
-import sys
+from functools import partial
import mozunit
import pytest
+from moztest.selftest.output import get_mozharness_status, filter_action
-from conftest import build, filter_action
-
-sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
from mozharness.base.log import INFO, WARNING, ERROR
-from mozharness.base.errors import BaseErrorList
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.mozilla.testing.errors import HarnessErrorList
-
-here = os.path.abspath(os.path.dirname(__file__))
-def get_mozharness_status(lines, status):
- parser = StructuredOutputParser(
- config={'log_level': INFO},
- error_list=BaseErrorList+HarnessErrorList,
- strict=False,
- suite_category='mochitest',
- )
-
- # Processing the log with mozharness will re-print all the output to stdout
- # Since this exact same output has already been printed by the actual test
- # run, temporarily redirect stdout to devnull.
- with open(os.devnull, 'w') as fh:
- orig = sys.stdout
- sys.stdout = fh
- for line in lines:
- parser.parse_single_line(json.dumps(line))
- sys.stdout = orig
- return parser.evaluate_parser(status)
+here = os.path.abspath(os.path.dirname(__file__))
+get_mozharness_status = partial(get_mozharness_status, 'mochitest')
def test_output_pass(runtests):
status, lines = runtests('test_pass.html')
assert status == 0
tbpl_status, log_level = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
--- a/testing/mochitest/tests/python/test_get_active_tests.py
+++ b/testing/mochitest/tests/python/test_get_active_tests.py
@@ -6,20 +6,22 @@ from __future__ import print_function, u
import os
from argparse import Namespace
from manifestparser import TestManifest
import mozunit
import pytest
+from conftest import setup_args
@pytest.fixture
-def get_active_tests(setup_harness_root, parser):
+def get_active_tests(setup_test_harness, parser):
+ setup_test_harness(*setup_args)
runtests = pytest.importorskip('runtests')
md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
options = vars(parser.parse_args([]))
def inner(**kwargs):
opts = options.copy()
opts.update(kwargs)
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/moztest/selftest/fixtures.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Pytest fixtures to help set up Firefox and a tests.zip
+in test harness selftests.
+"""
+
+import os
+import shutil
+import sys
+
+import mozfile
+import mozinstall
+import pytest
+import requests
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def _get_test_harness(suite, install_dir):
+ # Check if there is a local build
+ harness_root = os.path.join(build.topobjdir, '_tests', install_dir)
+ if os.path.isdir(harness_root):
+ return harness_root
+
+ # Check if it was previously set up by another test
+ harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', suite)
+ if os.path.isdir(harness_root):
+ return harness_root
+
+ # Check if there is a test package to download
+ if 'GECKO_INSTALLER_URL' in os.environ:
+ base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+ test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+ dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+ for name in test_packages[suite]:
+ url = base_url + '/' + name
+ bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+ r = requests.get(url, stream=True)
+ with open(bundle, 'w+b') as fh:
+ for chunk in r.iter_content(chunk_size=1024):
+ fh.write(chunk)
+
+ mozfile.extract(bundle, dest)
+
+ return os.path.join(dest, suite)
+
+ # Couldn't find a harness root, let caller do error handling.
+ return None
+
+
+@pytest.fixture(scope='session')
+def setup_test_harness(request):
+ """Fixture for setting up a mozharness-based test harness like
+ mochitest or reftest"""
+ def inner(files_dir, *args, **kwargs):
+ harness_root = _get_test_harness(*args, **kwargs)
+ if harness_root:
+ sys.path.insert(0, harness_root)
+
+ # Link the test files to the test package so updates are automatically
+ # picked up. Fallback to copy on Windows.
+ if files_dir:
+ test_root = os.path.join(harness_root, 'tests', 'selftests')
+ if not os.path.exists(test_root):
+ if hasattr(os, 'symlink'):
+ os.symlink(files_dir, test_root)
+ else:
+ shutil.copytree(files_dir, test_root)
+
+ elif 'GECKO_INSTALLER_URL' in os.environ:
+ # The mochitest tests will run regardless of whether a build exists or not.
+ # In a local environment, they should simply be skipped if setup fails. But
+ # in automation, we'll need to make sure an error is propagated up.
+ pytest.fail(HARNESS_ROOT_NOT_FOUND)
+ else:
+ # Tests will be marked skipped by the calls to pytest.importorskip() below.
+ # We are purposefully not failing here because running |mach python-test|
+ # without a build is a perfectly valid use case.
+ pass
+ return inner
+
+
+@pytest.fixture(scope='session')
+def binary():
+ """Return a Firefox binary"""
+ try:
+ return build.get_binary_path()
+ except:
+ pass
+
+ app = 'firefox'
+ bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
+ if os.path.isdir(bindir):
+ try:
+ return mozinstall.get_binary(bindir, app_name=app)
+ except:
+ pass
+
+ if 'GECKO_INSTALLER_URL' in os.environ:
+ bindir = mozinstall.install(
+ os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
+ return mozinstall.get_binary(bindir, app_name='firefox')
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/moztest/selftest/output.py
@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Methods for testing interactions with mozharness."""
+
+import json
+import os
+import sys
+
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
+from mozharness.base.log import INFO
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
+
+def get_mozharness_status(suite, lines, status):
+ """Given list of log lines, determine what the mozharness status would be."""
+ parser = StructuredOutputParser(
+ config={'log_level': INFO},
+ error_list=BaseErrorList+HarnessErrorList,
+ strict=False,
+ suite_category=suite,
+ )
+
+ # Processing the log with mozharness will re-print all the output to stdout
+ # Since this exact same output has already been printed by the actual test
+ # run, temporarily redirect stdout to devnull.
+ with open(os.devnull, 'w') as fh:
+ orig = sys.stdout
+ sys.stdout = fh
+ for line in lines:
+ parser.parse_single_line(json.dumps(line))
+ sys.stdout = orig
+ return parser.evaluate_parser(status)
+
+
+def filter_action(actions, lines):
+ if isinstance(actions, basestring):
+ actions = (actions,)
+ return filter(lambda x: x['action'] in actions, lines)