Bug 1335873 - Convert marionette harness unittests to standard python unittests, r?maja_zf draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 15 Feb 2017 16:38:45 -0500
changeset 491143 45e481fe4caef53472bf56d40c13158dd2eb3c20
parent 491142 a00518211f086e72ca724533b13a276bb7d46457
child 491144 e6a67dd12c5a1c3a09ec2c7bcc4091085a302035
push id47332
push userahalberstadt@mozilla.com
push dateWed, 01 Mar 2017 20:43:26 +0000
reviewersmaja_zf
bugs1335873
milestone54.0a1
Bug 1335873 - Convert marionette harness unittests to standard python unittests, r?maja_zf This formats the marionette-harness python tests to be a regular |mach python-test| suite. Though we add subsuite=marionette, this is just for automation purposes. The new preferred way to run the marionette harness tests locally is: ./mach python-test testing/marionette They will also run if running the full suite. The mozbase packages.txt file modifies mozlog to use 'setup.py' instead of 'pth'. The reason for this is that the marionette-harness tests use the pytest_mozlog pytest plugin for formatting their results (converts pytest format into something resembling the standard tbpl logging format). In order for this plugin to get picked up however, mozlog's setup.py file needs to be processed. MozReview-Commit-ID: Ata99evHxbd
python/mach_commands.py
taskcluster/ci/marionette-harness/kind.yml
taskcluster/ci/source-test/python-tests.yml
taskcluster/docs/kinds.rst
taskcluster/scripts/tester/harness-test-linux.sh
taskcluster/taskgraph/try_option_syntax.py
testing/config/marionette_harness_test_requirements.txt
testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
testing/mozbase/packages.txt
testing/mozharness/scripts/marionette_harness_tests.py
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -52,21 +52,16 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--verbose',
         default=False,
         action='store_true',
         help='Verbose output.')
     @CommandArgument('--stop',
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
-    @CommandArgument('--path-only',
-        default=False,
-        action='store_true',
-        help=('Collect all tests under given path instead of default '
-              'test resolution. Supports pytest-style tests.'))
     @CommandArgument('-j', '--jobs',
         default=1,
         type=int,
         help='Number of concurrent jobs to run. Default is 1.')
     @CommandArgument('--subsuite',
         default=None,
         help=('Python subsuite to run. If not specified, all subsuites are run. '
              'Use the string `default` to only run tests without a subsuite.'))
@@ -74,17 +69,16 @@ class MachCommands(MachCommandBase):
         metavar='TEST',
         help=('Tests to run. Each test can be a single file or a directory. '
               'Default test resolution relies on PYTHON_UNITTEST_MANIFESTS.'))
     def python_test(self,
                     tests=[],
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
-                    path_only=False,
                     stop=False,
                     jobs=1):
         self._activate_virtualenv()
 
         def find_tests_by_path():
             import glob
             files = []
             for t in tests:
@@ -104,40 +98,29 @@ class MachCommands(MachCommandBase):
 
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
         # is a simple way to keep environments separate, at the price of
         # launching Python multiple times. Most tests are run via mozunit,
         # which produces output in the format Mozilla infrastructure expects.
         # Some tests are run via pytest.
         if test_objects is None:
-            # If we're not being called from `mach test`, do our own
-            # test resolution.
-            if path_only:
-                if tests:
-                    test_objects = [{'path': p} for p in find_tests_by_path()]
-                else:
-                    self.log(logging.WARN, 'python-test', {},
-                             'TEST-UNEXPECTED-FAIL | No tests specified')
-                    test_objects = []
+            from mozbuild.testing import TestResolver
+            resolver = self._spawn(TestResolver)
+            if tests:
+                # If we were given test paths, try to find tests matching them.
+                test_objects = resolver.resolve_tests(paths=tests,
+                                                      flavor='python')
             else:
-                from mozbuild.testing import TestResolver
-                resolver = self._spawn(TestResolver)
-                if tests:
-                    # If we were given test paths, try to find tests matching them.
-                    test_objects = resolver.resolve_tests(paths=tests,
-                                                          flavor='python')
-                else:
-                    # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
-                    test_objects = resolver.resolve_tests(flavor='python')
+                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
+                test_objects = resolver.resolve_tests(flavor='python')
 
         if not test_objects:
-            message = 'TEST-UNEXPECTED-FAIL | No tests collected'
-            if not path_only:
-                message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
+            message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
+                      '(Not in PYTHON_UNITTEST_MANIFESTS?)'
             self.log(logging.WARN, 'python-test', {}, message)
             return 1
 
         mp = TestManifest()
         mp.tests.extend(test_objects)
 
         filters = []
         if subsuite == 'default':
deleted file mode 100644
--- a/taskcluster/ci/marionette-harness/kind.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
-
-implementation: taskgraph.task.transform:TransformTask
-
-transforms:
-   - taskgraph.transforms.marionette_harness:transforms
-   - taskgraph.transforms.task:transforms
-
-# NOTE: this task should be refactored so that it is invoked as a job either
-# with a run.using of "mozharness", and combined with the source-check kind.
-
-jobs:
-    marionette-harness/opt:
-        description: "Marionette harness unit test"
-        attributes:
-            build_platform: marionette-harness
-            build_type: opt
-        treeherder:
-            platform: linux64/opt
-            kind: test
-            tier: 2
-            symbol: tc(Mn-h)
-        worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-        worker:
-            implementation: docker-worker
-            docker-image: {in-tree: desktop-build}  # NOTE: better to use the lint image
-            env:
-                JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh"
-                MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py"
-                TOOLS_DISABLE: "true"
-            artifacts:
-              - name: public/logs/
-                path: /home/worker/workspace/mozharness_workspace/upload/logs/
-                type: directory
-            command:
-              - "bash"
-              - "/home/worker/bin/build.sh"
-              - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit"
-              - "--work-dir=mozharness_workspace"
-            max-run-time: 1800
-        when:
-            files-changed:
-              - "testing/marionette/harness/**"
-              - "testing/mozbase/mozlog/mozlog/pytest_mozlog/**"
-              - "testing/mozharness/scripts/marionette_harness_tests.py"
-              - "testing/config/marionette_harness_test_requirements.txt"
--- a/taskcluster/ci/source-test/python-tests.yml
+++ b/taskcluster/ci/source-test/python-tests.yml
@@ -17,16 +17,45 @@ taskgraph-tests/opt:
         - integration
         - release
     when:
         files-changed:
             - 'taskcluster/**/*.py'
             - 'config/mozunit.py'
             - 'python/mach/**/*.py'
 
+marionette-harness/opt:
+    description: testing/marionette/harness unit tests
+    platforms:
+        - linux64/opt
+    treeherder:
+        symbol: py(mnh)
+        kind: test
+        tier: 2
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/b2gtest
+    worker:
+        by-platform:
+            linux64.*:
+                implementation: docker-worker
+                docker-image: {in-tree: "lint"}
+                max-run-time: 3600
+    run:
+        using: mach
+        mach: python-test --subsuite marionette-harness
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+          - 'testing/marionette/harness/**'
+          - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
+          - 'python/mach_commands.py'
+
 mozbase/opt:
     description: testing/mozbase unit tests
     platforms:
         - linux64/opt
     treeherder:
         symbol: py(mb)
         kind: test
         tier: 2
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -85,21 +85,16 @@ will eventually be dependencies of the b
 are run manually via try pushes and the results uploaded to tooltool.
 
 spidermonkey
 ------------
 
 Spidermonkey tasks check out the full gecko source tree, then compile only the
 spidermonkey portion.  Each task runs specific tests after the build.
 
-marionette-harness
-------------------
-
-TBD (Maja)
-
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
deleted file mode 100644
--- a/taskcluster/scripts/tester/harness-test-linux.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#! /bin/bash -vex
-
-set -x -e
-
-echo "running as" $(id)
-
-####
-# Taskcluster friendly wrapper for running a script in
-# testing/mozharness/scripts in a source checkout (no build).
-# Example use: Python-only harness unit tests
-####
-
-: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
-: SRC_ROOT                      ${SRC_ROOT:=$WORKSPACE/build/src}
-# These paths should be relative to $SRC_ROOT
-: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
-: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
-: mozharness args               "${@}"
-
-set -v
-cd $WORKSPACE
-
-fail() {
-    echo # make sure error message is on a new line
-    echo "[harness-test-linux.sh:error]" "${@}"
-    exit 1
-}
-
-if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
-
-# support multiple, space delimited, config files
-config_cmds=""
-for cfg in $MOZHARNESS_CONFIG; do
-  config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}"
-done
-
-python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
-
-
-
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -32,17 +32,16 @@ BUILD_KINDS = set([
     'static-analysis',
     'spidermonkey',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-test',
     'toolchain',
-    'marionette-harness',
     'android-stuff',
 ])
 
 
 # mapping from shortcut name (usable with -u) to a boolean function identifying
 # matching test names
 def alias_prefix(prefix):
     return lambda name: name.startswith(prefix)
deleted file mode 100644
--- a/testing/config/marionette_harness_test_requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
--r mozbase_requirements.txt
-
-# TODO - if we structure common.tests.zip to match the in-tree structure of the
-# testing directory, we could use ./marionette_requirements.txt instead
-../web-platform/tests/tools/wptserve
-../marionette/client
-../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
-../marionette/harness
-
-# pytest
-../../python/py
-../../python/pytest
-../../python/mock-1.0.0
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini
@@ -0,0 +1,9 @@
+[DEFAULT]
+subsuite = marionette-harness
+
+[test_httpd.py]
+[test_marionette_arguments.py]
+[test_marionette_harness.py]
+[test_marionette_runner.py]
+[test_marionette_test_result.py]
+[test_serve.py]
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py
@@ -82,9 +82,10 @@ def test_handler(server):
     url = server.get_url("/httpd/test_handler")
     body = urllib2.urlopen(url).read()
     res = json.loads(body)
     assert res["count"] == counter
 
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(["--verbose", __file__]))
+    sys.exit(pytest.main(
+        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py
@@ -24,9 +24,10 @@ def test_parse_arg_socket_timeout(socket
         assert ex.value.code == 2
     else:
         args = parser.parse_args(args=argv)
         assert hasattr(args, 'socket_timeout') and args.socket_timeout == float(socket_timeout)
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(['--verbose', __file__]))
+    sys.exit(pytest.main(
+        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py
@@ -100,9 +100,10 @@ def test_harness_sets_up_default_test_ha
     harness = MarionetteHarness(args=mach_parsed_kwargs)
     mach_parsed_kwargs.pop('tests')
     runner = harness._runner_class(**mach_parsed_kwargs)
     assert marionette_test.MarionetteTestCase in runner.test_handlers
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(['--verbose', __file__]))
+    sys.exit(pytest.main(
+        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -434,9 +434,10 @@ def test_e10s_option_sets_prefs(mach_par
 def test_e10s_option_clash_raises(mock_runner):
     mock_runner.e10s = False
     with pytest.raises(AssertionError) as e:
         mock_runner.run_tests([u'test_fake_thing.py'])
         assert "configuration (self.e10s) does not match browser appinfo" in e.value.message
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(['--verbose', __file__]))
+    sys.exit(pytest.main(
+        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py
@@ -46,9 +46,10 @@ def test_crash_is_recorded_as_error(empt
     if has_crashed:
         assert len(result.errors) == 1
     else:
         assert len(result.errors) == 0
 
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(['--verbose', __file__]))
+    sys.exit(pytest.main(
+        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py
@@ -59,9 +59,10 @@ def test_iter_url():
 def test_where_is():
     serve.start()
     assert serve.where_is("/") == serve.servers["http"][1].get_url("/")
     assert serve.where_is("/", on="https") == serve.servers["https"][1].get_url("/")
 
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(["-s", "--verbose", __file__]))
+    sys.exit(pytest.main(
+        ['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
--- a/testing/mozbase/packages.txt
+++ b/testing/mozbase/packages.txt
@@ -3,17 +3,17 @@ mozb2g.pth:testing/mozbase/mozb2g
 mozcrash.pth:testing/mozbase/mozcrash
 mozdebug.pth:testing/mozbase/mozdebug
 mozdevice.pth:testing/mozbase/mozdevice
 mozfile.pth:testing/mozbase/mozfile
 mozhttpd.pth:testing/mozbase/mozhttpd
 mozinfo.pth:testing/mozbase/mozinfo
 mozinstall.pth:testing/mozbase/mozinstall
 mozleak.pth:testing/mozbase/mozleak
-mozlog.pth:testing/mozbase/mozlog
+setup.py:testing/mozbase/mozlog:develop
 moznetwork.pth:testing/mozbase/moznetwork
 mozprocess.pth:testing/mozbase/mozprocess
 mozprofile.pth:testing/mozbase/mozprofile
 mozrunner.pth:testing/mozbase/mozrunner
 mozsystemmonitor.pth:testing/mozbase/mozsystemmonitor
 mozscreenshot.pth:testing/mozbase/mozscreenshot
 moztest.pth:testing/mozbase/moztest
 mozversion.pth:testing/mozbase/mozversion
deleted file mode 100644
--- a/testing/mozharness/scripts/marionette_harness_tests.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-import copy
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.python import PreScriptAction
-from mozharness.base.python import (
-    VirtualenvMixin,
-    virtualenv_config_options,
-)
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import (
-    BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE,
-    TBPL_EXCEPTION
-)
-
-marionette_harness_tests_config_options = [
-    [['--tests'], {
-        'dest': 'test_path',
-        'default': None,
-        'help': 'Path to test_*.py or directory relative to src root.',
-    }],
-    [['--src-dir'], {
-        'dest': 'rel_src_dir',
-        'default': None,
-        'help': 'Path to hg.mo source checkout relative to work dir.',
-    }],
-
-] + copy.deepcopy(virtualenv_config_options)
-
-marionette_harness_tests_config = {
-    "find_links": [
-        "http://pypi.pub.build.mozilla.org/pub",
-    ],
-    "pip_index": False,
-    # relative to workspace
-    "rel_src_dir": os.path.join("build", "src"),
-}
-
-class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript):
-
-    def __init__(self, config_options=None,
-                 all_actions=None, default_actions=None,
-                 *args, **kwargs):
-        config_options = config_options or marionette_harness_tests_config_options
-        actions = [
-            'clobber',
-            'create-virtualenv',
-            'run-tests',
-        ]
-        super(MarionetteHarnessTests, self).__init__(
-            config_options=config_options,
-            all_actions=all_actions or actions,
-            default_actions=default_actions or actions,
-            config=marionette_harness_tests_config,
-            *args, **kwargs)
-
-    @PreScriptAction('create-virtualenv')
-    def _pre_create_virtualenv(self, action):
-        dirs = self.query_abs_dirs()
-        c = self.config
-        requirements = os.path.join(
-            dirs['abs_src_dir'],
-            'testing', 'config',
-            'marionette_harness_test_requirements.txt'
-        )
-        self.register_virtualenv_module(
-           requirements=[requirements],
-           two_pass=True
-        )
-
-    def query_abs_dirs(self):
-        if self.abs_dirs:
-            return self.abs_dirs
-        c = self.config
-        abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs()
-        dirs = {
-            'abs_src_dir': os.path.abspath(
-                os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir'])
-            ),
-        }
-
-        for key in dirs:
-            if key not in abs_dirs:
-                abs_dirs[key] = dirs[key]
-        self.abs_dirs = abs_dirs
-
-        return self.abs_dirs
-
-    def _get_pytest_status(self, code):
-        """
-        Translate pytest exit code to TH status
-
-        Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26
-        """
-        if code == 0:
-            return TBPL_SUCCESS
-        elif code == 1:
-            return TBPL_WARNING
-        elif 1 < code < 6:
-            self.error("pytest returned exit code: %s" % code)
-            return TBPL_FAILURE
-        else:
-            return TBPL_EXCEPTION
-
-    def run_tests(self):
-        """Run all the tests"""
-        dirs = self.query_abs_dirs()
-        test_relpath = self.config.get(
-            'test_path',
-            os.path.join('testing', 'marionette',
-                         'harness', 'marionette_harness', 'tests',
-                         'harness_unit')
-        )
-        test_path = os.path.join(dirs['abs_src_dir'], test_relpath)
-        self.activate_virtualenv()
-        import pytest
-        command = ['-p', 'no:terminalreporter',  # disable pytest logging
-                   test_path]
-        logs = {}
-        for fmt in ['tbpl', 'mach', 'raw']:
-            logs[fmt] = os.path.join(dirs['abs_log_dir'],
-                                     'mn-harness_{}.log'.format(fmt))
-            command.extend(['--log-'+fmt, logs[fmt]])
-        self.info('Calling pytest.main with the following arguments: %s' % command)
-        status = self._get_pytest_status(pytest.main(command))
-        self.read_from_file(logs['tbpl'])
-        for log in logs.values():
-            self.copy_to_upload_dir(log, dest='logs/')
-        self.buildbot_status(status)
-
-
-if __name__ == '__main__':
-    script = MarionetteHarnessTests()
-    script.run_and_exit()