Bug 1392390 - Refactor common code out of mochitest selftests and into a new moztest.selftest module, r?jmaher draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 11 Sep 2017 16:06:06 -0400
changeset 663024 a4b9e1f2edb14016744a834541c14b050af1ae71
parent 662290 f9a5e9ed62103c84e4cde915f4d08f1ce71be83e
child 663025 84b7628f7e42d0bea7f2e0ea2d69a82ae807edbe
push id79294
push userahalberstadt@mozilla.com
push dateTue, 12 Sep 2017 14:34:46 +0000
reviewersjmaher
bugs1392390
milestone57.0a1
Bug 1392390 - Refactor common code out of mochitest selftests and into a new moztest.selftest module, r?jmaher This includes code for downloading a Firefox binary, downloading + setting up a tests.zip and running output through mozharness' output parsers. This is all stuff that will also be required for the reftest selftests. I couldn't think of a better location to put this stuff, suggestions welcome. MozReview-Commit-ID: 59TSbsugT5T
taskcluster/ci/source-test/python.yml
testing/mochitest/tests/python/conftest.py
testing/mochitest/tests/python/test_basic_mochitest_plain.py
testing/mochitest/tests/python/test_get_active_tests.py
testing/mozbase/moztest/moztest/selftest/__init__.py
testing/mozbase/moztest/moztest/selftest/fixtures.py
testing/mozbase/moztest/moztest/selftest/output.py
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -70,16 +70,17 @@ mochitest-harness:
             start_xvfb '1600x1200x24' 0 &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --subsuite mochitest
     when:
         files-changed:
             - 'config/mozunit.py'
             - 'python/mach_commands.py'
             - 'testing/mochitest/**'
+            - 'testing/mozbase/moztest/moztest/selftest/**'
             - 'testing/mozharness/mozharness/base/log.py'
             - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
             - 'testing/mozharness/mozharness/mozilla/testing/errors.py'
             - 'testing/profiles/prefs_general.js'
 
 mozbase:
     description: testing/mozbase unit tests
     platform:
--- a/testing/mochitest/tests/python/conftest.py
+++ b/testing/mochitest/tests/python/conftest.py
@@ -1,144 +1,45 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
-
 import json
 import os
-import shutil
-import sys
 from argparse import Namespace
 from cStringIO import StringIO
 
 import pytest
-import requests
 
-import mozfile
 import mozinfo
-import mozinstall
 from manifestparser import TestManifest, expression
-from mozbuild.base import MozbuildObject
+from moztest.selftest.fixtures import binary, setup_test_harness  # noqa
 
 here = os.path.abspath(os.path.dirname(__file__))
-build = MozbuildObject.from_environment(cwd=here)
-
-HARNESS_ROOT_NOT_FOUND = """
-Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
-environment variable is required.
-""".lstrip()
-
-
-def filter_action(actions, lines):
-    if isinstance(actions, basestring):
-        actions = (actions,)
-    return filter(lambda x: x['action'] in actions, lines)
-
-
-def _get_harness_root():
-    # Check if there is a local build
-    harness_root = os.path.join(build.topobjdir, '_tests', 'testing', 'mochitest')
-    if os.path.isdir(harness_root):
-        return harness_root
-
-    # Check if it was previously set up by another test
-    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', 'mochitest')
-    if os.path.isdir(harness_root):
-        return harness_root
-
-    # Check if there is a test package to download
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
-        test_packages = requests.get(base_url + '/target.test_packages.json').json()
-
-        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
-        for name in test_packages['mochitest']:
-            url = base_url + '/' + name
-            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
-
-            r = requests.get(url, stream=True)
-            with open(bundle, 'w+b') as fh:
-                for chunk in r.iter_content(chunk_size=1024):
-                    fh.write(chunk)
-
-            mozfile.extract(bundle, dest)
-
-        return os.path.join(dest, 'mochitest')
-
-    # Couldn't find a harness root, let caller do error handling.
-    return None
-
-
-@pytest.fixture(scope='session')
-def setup_harness_root():
-    harness_root = _get_harness_root()
-    if harness_root:
-        sys.path.insert(0, harness_root)
-
-        # Link the test files to the test package so updates are automatically
-        # picked up. Fallback to copy on Windows.
-        test_root = os.path.join(harness_root, 'tests', 'selftests')
-        if not os.path.exists(test_root):
-            files = os.path.join(here, 'files')
-            if hasattr(os, 'symlink'):
-                os.symlink(files, test_root)
-            else:
-                shutil.copytree(files, test_root)
-
-    elif 'GECKO_INSTALLER_URL' in os.environ:
-        # The mochitest tests will run regardless of whether a build exists or not.
-        # In a local environment, they should simply be skipped if setup fails. But
-        # in automation, we'll need to make sure an error is propagated up.
-        pytest.fail(HARNESS_ROOT_NOT_FOUND)
-    else:
-        # Tests will be marked skipped by the calls to pytest.importorskip() below.
-        # We are purposefully not failing here because running |mach python-test|
-        # without a build is a perfectly valid use case.
-        pass
-
-
-@pytest.fixture(scope='session')
-def binary():
-    try:
-        return build.get_binary_path()
-    except:
-        pass
-
-    app = 'firefox'
-    bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
-    if os.path.isdir(bindir):
-        try:
-            return mozinstall.get_binary(bindir, app_name=app)
-        except:
-            pass
-
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        bindir = mozinstall.install(
-            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
-        return mozinstall.get_binary(bindir, app_name='firefox')
+setup_args = [os.path.join(here, 'files'), 'mochitest', 'testing/mochitest']
 
 
 @pytest.fixture(scope='function')
 def parser(request):
     parser = pytest.importorskip('mochitest_options')
 
     app = getattr(request.module, 'APP', 'generic')
     return parser.MochitestArgumentParser(app=app)
 
 
-@pytest.fixture(scope='function')
-def runtests(setup_harness_root, binary, parser, request):
+@pytest.fixture(scope='function')  # noqa: F811
+def runtests(setup_test_harness, binary, parser, request):
     """Creates an easy to use entry point into the mochitest harness.
 
     :returns: A function with the signature `*tests, **opts`. Each test is a file name
               (relative to the `files` dir). At least one is required. The opts are
               used to override the default mochitest options, they are optional.
     """
+    setup_test_harness(*setup_args)
     runtests = pytest.importorskip('runtests')
 
     mochitest_root = runtests.SCRIPT_DIR
     test_root = os.path.join(mochitest_root, 'tests', 'selftests')
 
     buf = StringIO()
     options = vars(parser.parse_args([]))
     options.update({
@@ -176,32 +77,34 @@ def runtests(setup_harness_root, binary,
 
         result = runtests.run_test_harness(parser, Namespace(**options))
         out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
         buf.close()
         return result, out
     return inner
 
 
-@pytest.fixture
-def build_obj(setup_harness_root):
+@pytest.fixture  # noqa: F811
+def build_obj(setup_test_harness):
+    setup_test_harness(*setup_args)
     mochitest_options = pytest.importorskip('mochitest_options')
     return mochitest_options.build_obj
 
 
-@pytest.fixture(autouse=True)
-def skip_using_mozinfo(request, setup_harness_root):
+@pytest.fixture(autouse=True)  # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
     """Gives tests the ability to skip based on values from mozinfo.
 
     Example:
         @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
         def test_foo():
             pass
     """
 
+    setup_test_harness(*setup_args)
     runtests = pytest.importorskip('runtests')
     runtests.update_mozinfo()
 
     skip_mozinfo = request.node.get_marker('skip_mozinfo')
     if skip_mozinfo:
         value = skip_mozinfo.args[0]
         if expression.parse(value, **mozinfo.info):
             pytest.skip("skipped due to mozinfo match: \n{}".format(value))
--- a/testing/mochitest/tests/python/test_basic_mochitest_plain.py
+++ b/testing/mochitest/tests/python/test_basic_mochitest_plain.py
@@ -1,49 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import json
 import os
-import sys
+from functools import partial
 
 import mozunit
 import pytest
+from moztest.selftest.output import get_mozharness_status, filter_action
 
-from conftest import build, filter_action
-
-sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
 from mozharness.base.log import INFO, WARNING, ERROR
-from mozharness.base.errors import BaseErrorList
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.mozilla.testing.errors import HarnessErrorList
-
-here = os.path.abspath(os.path.dirname(__file__))
 
 
-def get_mozharness_status(lines, status):
-    parser = StructuredOutputParser(
-        config={'log_level': INFO},
-        error_list=BaseErrorList+HarnessErrorList,
-        strict=False,
-        suite_category='mochitest',
-    )
-
-    # Processing the log with mozharness will re-print all the output to stdout
-    # Since this exact same output has already been printed by the actual test
-    # run, temporarily redirect stdout to devnull.
-    with open(os.devnull, 'w') as fh:
-        orig = sys.stdout
-        sys.stdout = fh
-        for line in lines:
-            parser.parse_single_line(json.dumps(line))
-        sys.stdout = orig
-    return parser.evaluate_parser(status)
+here = os.path.abspath(os.path.dirname(__file__))
+get_mozharness_status = partial(get_mozharness_status, 'mochitest')
 
 
 def test_output_pass(runtests):
     status, lines = runtests('test_pass.html')
     assert status == 0
 
     tbpl_status, log_level = get_mozharness_status(lines, status)
     assert tbpl_status == TBPL_SUCCESS
--- a/testing/mochitest/tests/python/test_get_active_tests.py
+++ b/testing/mochitest/tests/python/test_get_active_tests.py
@@ -6,20 +6,22 @@ from __future__ import print_function, u
 
 import os
 from argparse import Namespace
 
 from manifestparser import TestManifest
 
 import mozunit
 import pytest
+from conftest import setup_args
 
 
 @pytest.fixture
-def get_active_tests(setup_harness_root, parser):
+def get_active_tests(setup_test_harness, parser):
+    setup_test_harness(*setup_args)
     runtests = pytest.importorskip('runtests')
     md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
 
     options = vars(parser.parse_args([]))
 
     def inner(**kwargs):
         opts = options.copy()
         opts.update(kwargs)
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/moztest/selftest/fixtures.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Pytest fixtures to help set up Firefox and a tests.zip
+in test harness selftests.
+"""
+
+import os
+import shutil
+import sys
+
+import mozfile
+import mozinstall
+import pytest
+import requests
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+
+HARNESS_ROOT_NOT_FOUND = """
+Could not find test harness root. Either a build or the 'GECKO_INSTALLER_URL'
+environment variable is required.
+""".lstrip()
+
+
+def _get_test_harness(suite, install_dir):
+    # Check if there is a local build
+    harness_root = os.path.join(build.topobjdir, '_tests', install_dir)
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if it was previously set up by another test
+    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', suite)
+    if os.path.isdir(harness_root):
+        return harness_root
+
+    # Check if there is a test package to download
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
+        test_packages = requests.get(base_url + '/target.test_packages.json').json()
+
+        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
+        for name in test_packages[suite]:
+            url = base_url + '/' + name
+            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
+
+            r = requests.get(url, stream=True)
+            with open(bundle, 'w+b') as fh:
+                for chunk in r.iter_content(chunk_size=1024):
+                    fh.write(chunk)
+
+            mozfile.extract(bundle, dest)
+
+        return os.path.join(dest, suite)
+
+    # Couldn't find a harness root, let caller do error handling.
+    return None
+
+
+@pytest.fixture(scope='session')
+def setup_test_harness(request):
+    """Fixture for setting up a mozharness-based test harness like
+    mochitest or reftest"""
+    def inner(files_dir, *args, **kwargs):
+        harness_root = _get_test_harness(*args, **kwargs)
+        if harness_root:
+            sys.path.insert(0, harness_root)
+
+            # Link the test files to the test package so updates are automatically
+            # picked up. Fallback to copy on Windows.
+            if files_dir:
+                test_root = os.path.join(harness_root, 'tests', 'selftests')
+                if not os.path.exists(test_root):
+                    if hasattr(os, 'symlink'):
+                        os.symlink(files_dir, test_root)
+                    else:
+                        shutil.copytree(files_dir, test_root)
+
+        elif 'GECKO_INSTALLER_URL' in os.environ:
+            # The mochitest tests will run regardless of whether a build exists or not.
+            # In a local environment, they should simply be skipped if setup fails. But
+            # in automation, we'll need to make sure an error is propagated up.
+            pytest.fail(HARNESS_ROOT_NOT_FOUND)
+        else:
+            # Tests will be marked skipped by the calls to pytest.importorskip() below.
+            # We are purposefully not failing here because running |mach python-test|
+            # without a build is a perfectly valid use case.
+            pass
+    return inner
+
+
+@pytest.fixture(scope='session')
+def binary():
+    """Return a Firefox binary"""
+    try:
+        return build.get_binary_path()
+    except:
+        pass
+
+    app = 'firefox'
+    bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
+    if os.path.isdir(bindir):
+        try:
+            return mozinstall.get_binary(bindir, app_name=app)
+        except:
+            pass
+
+    if 'GECKO_INSTALLER_URL' in os.environ:
+        bindir = mozinstall.install(
+            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
+        return mozinstall.get_binary(bindir, app_name='firefox')
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/moztest/moztest/selftest/output.py
@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Methods for testing interactions with mozharness."""
+
+import json
+import os
+import sys
+
+from mozbuild.base import MozbuildObject
+
+here = os.path.abspath(os.path.dirname(__file__))
+build = MozbuildObject.from_environment(cwd=here)
+
+sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
+from mozharness.base.log import INFO
+from mozharness.base.errors import BaseErrorList
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+from mozharness.mozilla.testing.errors import HarnessErrorList
+
+
+def get_mozharness_status(suite, lines, status):
+    """Given list of log lines, determine what the mozharness status would be."""
+    parser = StructuredOutputParser(
+        config={'log_level': INFO},
+        error_list=BaseErrorList+HarnessErrorList,
+        strict=False,
+        suite_category=suite,
+    )
+
+    # Processing the log with mozharness will re-print all the output to stdout
+    # Since this exact same output has already been printed by the actual test
+    # run, temporarily redirect stdout to devnull.
+    with open(os.devnull, 'w') as fh:
+        orig = sys.stdout
+        sys.stdout = fh
+        for line in lines:
+            parser.parse_single_line(json.dumps(line))
+        sys.stdout = orig
+    return parser.evaluate_parser(status)
+
+
+def filter_action(actions, lines):
+    if isinstance(actions, basestring):
+        actions = (actions,)
+    return filter(lambda x: x['action'] in actions, lines)