Bug 1191324 - Extend Marionette to allow automation of telemetry tests; r?maja_zf draft
authorAndré Reinald <areinald@mozilla.com>
Thu, 14 Apr 2016 18:34:28 +0200
changeset 351010 68abdb2e234cb17763a1b5f3aa1c692245c5b384
parent 349408 5329e23d3b0cbeef7cb22533578bb09d08b2a763
child 518450 ebcc54624f0094ddb3676f2f23576b721790e6e2
push id15470
push userareinald@mozilla.com
push dateThu, 14 Apr 2016 16:41:07 +0000
reviewersmaja_zf
bugs1191324
milestone48.0a1
Bug 1191324 - Extend Marionette to allow automation of telemetry tests; r?maja_zf WIP - changed names of classes / methods / strings from "2" to "session" MozReview-Commit-ID: DrZyaBKmaNN
testing/marionette/harness/marionette/__init__.py
testing/marionette/harness/session/__init__.py
testing/marionette/harness/session/runner/__init__.py
testing/marionette/harness/session/runner/base.py
testing/marionette/harness/session/runner/base2.py
testing/marionette/harness/session/runtests.py
testing/marionette/harness/session/session_test.py
testing/marionette/harness/session/tests/test_marionette2.py
testing/marionette/harness/session/tests/test_session.py
testing/marionette/harness/session/tests/unit-tests.ini
testing/marionette/mach_commands.py
--- a/testing/marionette/harness/marionette/__init__.py
+++ b/testing/marionette/harness/marionette/__init__.py
@@ -9,20 +9,17 @@ from .marionette_test import (
     expectedFailure,
     MarionetteJSTestCase,
     MarionetteTestCase,
     skip,
     skip_if_b2g,
     SkipTest,
     skip_unless_protocol,
 )
-from .marionette2_test import (
-    Marionette2JSTestCase,
-    Marionette2TestCase,
-)
+
 from .runner import (
     B2GTestCaseMixin,
     B2GTestResultMixin,
     BaseMarionetteArguments,
     BaseMarionetteTestRunner,
     BaseMarionette2Arguments,
     BaseMarionette2TestRunner,
     BrowserMobProxyTestCaseMixin,
copy from testing/marionette/harness/marionette/__init__.py
copy to testing/marionette/harness/session/__init__.py
--- a/testing/marionette/harness/marionette/__init__.py
+++ b/testing/marionette/harness/session/__init__.py
@@ -1,36 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 __version__ = '2.2.0'
 
-from .marionette_test import (
+from .session_test import (
     CommonTestCase,
     expectedFailure,
-    MarionetteJSTestCase,
-    MarionetteTestCase,
+    SessionJSTestCase,
+    SessionTestCase,
     skip,
     skip_if_b2g,
     SkipTest,
     skip_unless_protocol,
 )
-from .marionette2_test import (
-    Marionette2JSTestCase,
-    Marionette2TestCase,
-)
+
 from .runner import (
-    B2GTestCaseMixin,
-    B2GTestResultMixin,
-    BaseMarionetteArguments,
-    BaseMarionetteTestRunner,
-    BaseMarionette2Arguments,
-    BaseMarionette2TestRunner,
-    BrowserMobProxyTestCaseMixin,
+    BaseSessionArguments,
+    BaseSessionTestRunner,
     EnduranceArguments,
     EnduranceTestCaseMixin,
     HTMLReportingArguments,
     HTMLReportingTestResultMixin,
     HTMLReportingTestRunnerMixin,
     Marionette,
     MarionetteTest,
     MarionetteTestResult,
copy from testing/marionette/harness/marionette/runner/__init__.py
copy to testing/marionette/harness/session/runner/__init__.py
--- a/testing/marionette/harness/marionette/runner/__init__.py
+++ b/testing/marionette/harness/session/runner/__init__.py
@@ -1,35 +1,11 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from .base import (
-    B2GTestResultMixin,
-    BaseMarionetteArguments,
-    BaseMarionetteTestRunner,
-    Marionette,
-    MarionetteTest,
-    MarionetteTestResult,
-    MarionetteTextTestRunner,
-    TestManifest,
-    TestResult,
-    TestResultCollection,
+    BaseSessionArguments,
+    BaseSessionTestRunner,
+    SessionTest,
+    SessionTestResult,
+    SessionTextTestRunner,
 )
-
-from .base2 import (
-    BaseMarionette2Arguments,
-    BaseMarionette2TestRunner,
-)
-
-from .mixins import (
-    B2GTestCaseMixin,
-    B2GTestResultMixin,
-    EnduranceArguments,
-    EnduranceTestCaseMixin,
-    HTMLReportingArguments,
-    HTMLReportingTestResultMixin,
-    HTMLReportingTestRunnerMixin,
-    MemoryEnduranceTestCaseMixin,
-    BrowserMobProxyTestCaseMixin,
-    BrowserMobProxyArguments,
-    BrowserMobTestCase,
-)
rename from testing/marionette/harness/session/runner/base2.py
rename to testing/marionette/harness/session/runner/base.py
--- a/testing/marionette/harness/session/runner/base2.py
+++ b/testing/marionette/harness/session/runner/base.py
@@ -26,33 +26,33 @@ from moztest.results import TestResultCo
 import mozversion
 
 import httpd
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
-class Marionette2Test(TestResult):
+class SessionTest(TestResult):
 
     @property
     def test_name(self):
         if self.test_class is not None:
             return '%s.py %s.%s' % (self.test_class.split('.')[0],
                                     self.test_class,
                                     self.name)
         else:
             return self.name
 
-class Marionette2TestResult(StructuredTestResult, TestResultCollection):
+class SessionTestResult(StructuredTestResult, TestResultCollection):
 
-    resultClass = Marionette2Test
+    resultClass = SessionTest
 
     def __init__(self, *args, **kwargs):
-        TestResultCollection.__init__(self, 'Marionette2Test')
+        TestResultCollection.__init__(self, 'SessionTest')
         self.passed = 0
         self.testsRun = 0
         self.result_modifiers = [] # used by mixins to modify the result
         logcat_stdout = kwargs.pop('logcat_stdout')
         StructuredTestResult.__init__(self, *args, **kwargs)
 
     @property
     def skipped(self):
@@ -125,41 +125,41 @@ class Marionette2TestResult(StructuredTe
         t.finish(result_actual,
                  time_end=time.time() if test.start_time else 0,
                  reason=relevant_line(output),
                  output=output)
         self.append(t)
 
     def addError(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='ERROR')
-        super(Marionette2TestResult, self).addError(test, err)
+        super(SessionTestResult, self).addError(test, err)
 
     def addFailure(self, test, err):
         self.add_test_result(test, output=self._exc_info_to_string(err, test), result_actual='UNEXPECTED-FAIL')
-        super(Marionette2TestResult, self).addFailure(test, err)
+        super(SessionTestResult, self).addFailure(test, err)
 
     def addSuccess(self, test):
         self.passed += 1
         self.add_test_result(test, result_actual='PASS')
-        super(Marionette2TestResult, self).addSuccess(test)
+        super(SessionTestResult, self).addSuccess(test)
 
     def addExpectedFailure(self, test, err):
         """Called when an expected failure/error occured."""
         self.add_test_result(test, output=self._exc_info_to_string(err, test),
                              result_actual='KNOWN-FAIL')
-        super(Marionette2TestResult, self).addExpectedFailure(test, err)
+        super(SessionTestResult, self).addExpectedFailure(test, err)
 
     def addUnexpectedSuccess(self, test):
         """Called when a test was expected to fail, but succeed."""
         self.add_test_result(test, result_actual='UNEXPECTED-PASS')
-        super(Marionette2TestResult, self).addUnexpectedSuccess(test)
+        super(SessionTestResult, self).addUnexpectedSuccess(test)
 
     def addSkip(self, test, reason):
         self.add_test_result(test, output=reason, result_actual='SKIPPED')
-        super(Marionette2TestResult, self).addSkip(test, reason)
+        super(SessionTestResult, self).addSkip(test, reason)
 
     def getInfo(self, test):
         return test.test_name
 
     def getDescription(self, test):
         doc_first_line = test.shortDescription()
         if self.descriptions and doc_first_line:
             return '\n'.join((str(test), doc_first_line))
@@ -187,19 +187,19 @@ class Marionette2TestResult(StructuredTe
                     self.logger.info(' '.join(line).encode('ascii', 'replace'))
                 self.logger.info('END LOG:')
 
     def stopTest(self, *args, **kwargs):
         unittest._TextTestResult.stopTest(self, *args, **kwargs)
         self.shouldStop = True # not sure here
 
 
-class Marionette2TextTestRunner(StructuredTestRunner):
+class SessionTextTestRunner(StructuredTestRunner):
 
-    resultclass = Marionette2TestResult
+    resultclass = SessionTestResult
 
     def __init__(self, **kwargs):
         self.pre_run_functions = []
         self.logcat_stdout = kwargs.pop('logcat_stdout')
         self.binary = kwargs.pop('binary')
 
         StructuredTestRunner.__init__(self, **kwargs)
 
@@ -212,22 +212,22 @@ class Marionette2TextTestRunner(Structur
                                 logcat_stdout=self.logcat_stdout,
                                 result_callbacks=self.result_callbacks)
 
     def run(self, test):
         "Run the given test case or test suite."
         for pre_run_func in self.pre_run_functions:
             pre_run_func()
 
-        result = super(Marionette2TextTestRunner, self).run(test)
+        result = super(SessionTextTestRunner, self).run(test)
         result.printLogs(test)
         return result
 
 
-class BaseMarionette2Arguments(ArgumentParser):
+class BaseSessionArguments(ArgumentParser):
     socket_timeout_default = 360.0
 
     def __init__(self, **kwargs):
         ArgumentParser.__init__(self, **kwargs)
 
         def dir_path(path):
             path = os.path.abspath(os.path.expanduser(path))
             if not os.access(path, os.F_OK):
@@ -416,32 +416,32 @@ class BaseMarionette2Arguments(ArgumentP
 
         for container in self.argument_containers:
             if hasattr(container, 'verify_usage_handler'):
                 container.verify_usage_handler(args)
 
         return args
 
 
-class BaseMarionette2TestRunner(object):
+class BaseSessionTestRunner(object):
 
-    textrunnerclass = Marionette2TextTestRunner
+    textrunnerclass = SessionTextTestRunner
     driverclass = Marionette
 
     def __init__(self, address=None,
                  homedir=None,
                  binary=None,
                  logger=None, logdir=None, logcat_stdout=False,
                  repeat=0, testvars=None, type=None,
                  symbols_path=None, timeout=None,
                  shuffle=False, shuffle_seed=random.randint(0, sys.maxint),
                  this_chunk=1, total_chunks=1, sources=None,
                  server_root=None, gecko_log=None, result_callbacks=None,
                  prefs=None, test_tags=None,
-                 socket_timeout=BaseMarionette2Arguments.socket_timeout_default,
+                 socket_timeout=BaseSessionArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, workspace=None,
                  verbose=0, **kwargs):
         self.address = address
         self.homedir = homedir
         self.bin = binary
         self.addons = addons
         self.logger = logger
         self.httpd = None
--- a/testing/marionette/harness/session/runtests.py
+++ b/testing/marionette/harness/session/runtests.py
@@ -1,39 +1,39 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 
 from marionette import __version__
 from marionette_driver import __version__ as driver_version
-from marionette.marionette2_test import Marionette2TestCase, Marionette2JSTestCase
-from marionette.runner import (
-    BaseMarionette2TestRunner,
-    BaseMarionette2Arguments,
+from session.session_test import SessionTestCase, SessionJSTestCase
+from session.runner import (
+    BaseSessionTestRunner,
+    BaseSessionArguments,
 )
 import mozlog
 
 
-class Marionette2TestRunner(BaseMarionette2TestRunner):
+class SessionTestRunner(BaseSessionTestRunner):
     def __init__(self, **kwargs):
-        BaseMarionette2TestRunner.__init__(self, **kwargs)
-        self.test_handlers = [Marionette2TestCase, Marionette2JSTestCase]
+        BaseSessionTestRunner.__init__(self, **kwargs)
+        self.test_handlers = [SessionTestCase, SessionJSTestCase]
 
 
-class Marionette2Arguments(BaseMarionette2Arguments):
+class SessionArguments(BaseSessionArguments):
     def __init__(self, **kwargs):
-        BaseMarionette2Arguments.__init__(self, **kwargs)
+        BaseSessionArguments.__init__(self, **kwargs)
 
 
-class Marionette2Harness(object):
+class SessionHarness(object):
     def __init__(self,
-                 runner_class=Marionette2TestRunner,
-                 parser_class=Marionette2Arguments,
+                 runner_class=SessionTestRunner,
+                 parser_class=SessionArguments,
                  args=None):
         self._runner_class = runner_class
         self._parser_class = parser_class
         self.args = args or self.parse_args()
 
     def parse_args(self, logger_defaults=None):
         parser = self._parser_class(usage='%(prog)s [options] test_file_or_dir <test_file_or_dir> ...')
         parser.add_argument('--version', action='version',
@@ -52,43 +52,43 @@ class Marionette2Harness(object):
         logger = mozlog.commandline.setup_logging(
             args.logger_name, args, logger_defaults or {"tbpl": sys.stdout})
 
         args.logger = logger
         return args
 
     def process_args(self):
         if self.args.pydebugger:
-            Marionette2TestCase.pydebugger = __import__(self.args.pydebugger)
+            SessionTestCase.pydebugger = __import__(self.args.pydebugger)
 
     def run(self):
         try:
             self.process_args()
             args_dict = vars(self.args)
             tests = args_dict.pop('tests')
             runner = self._runner_class(**args_dict)
             runner.run_tests(tests)
             return runner.failed
         except Exception:
             self.args.logger.error('Failure during test execution.',
                                    exc_info=True)
             raise
 
 
-def cli(runner_class=Marionette2TestRunner, parser_class=Marionette2Arguments,
-        harness_class=Marionette2Harness, args=None):
+def cli(runner_class=SessionTestRunner, parser_class=SessionArguments,
+        harness_class=SessionHarness, args=None):
     """
     Call the harness to parse args and run tests.
 
     The following exit codes are expected:
     - Test failures: 10
     - Harness/other failures: 1
     - Success: 0
     """
-    logger = mozlog.commandline.setup_logging('Marionette2 test runner', {})
+    logger = mozlog.commandline.setup_logging('Session test runner', {})
     try:
         failed = harness_class(runner_class, parser_class, args=args).run()
         if failed > 0:
             sys.exit(10)
     except Exception:
         logger.error('Failure during harness setup', exc_info=True)
         sys.exit(1)
     sys.exit(0)
--- a/testing/marionette/harness/session/session_test.py
+++ b/testing/marionette/harness/session/session_test.py
@@ -24,30 +24,30 @@ from marionette_driver.errors import (
         InvalidCookieDomainException, UnableToSetCookieException, InvalidSelectorException,
         MoveTargetOutOfBoundsException
         )
 from marionette_driver.marionette import Marionette
 from marionette_driver.wait import Wait
 from marionette_driver.expected import element_present, element_not_present
 from mozlog import get_default_logger
 
-from marionette_test import (
+from marionette.marionette_test import (
         SkipTest,
         _ExpectedFailure,
         _UnexpectedSuccess,
         skip,
         expectedFailure,
         parameterized,
         with_parameters,
         wraps_parameterized,
         MetaParameterized,
         JSTest
         )
 
-class Common2TestCase(unittest.TestCase):
+class CommonTestCase(unittest.TestCase):
 
     __metaclass__ = MetaParameterized
     match_re = None
     failureException = AssertionError
     pydebugger = None
 
     def __init__(self, methodName, **kwargs):
         unittest.TestCase.__init__(self, methodName)
@@ -392,27 +392,27 @@ class Common2TestCase(unittest.TestCase)
                 pass
             else:
                 self.loglines = marionette.get_logs()
                 raise
         self.marionette.test_name = original_test_name
 
 
 
-class Marionette2TestCase(Common2TestCase):
+class SessionTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.py$")
 
     def __init__(self, methodName='runTest',
                  filepath='', **kwargs):
         self.marionette = None
         self.methodName = methodName
         self.filepath = filepath
         self.testvars = kwargs.pop('testvars', None)
-        Common2TestCase.__init__(self, methodName, **kwargs)
+        CommonTestCase.__init__(self, methodName, **kwargs)
 
     @classmethod
     def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
         # since we use imp.load_source to load test modules, if a module
         # is loaded with the same name as another one the module would just be
         # reloaded.
         #
         # We may end up by finding too many test in a module then since
@@ -434,40 +434,40 @@ class Marionette2TestCase(Common2TestCas
                 testnames = testloader.getTestCaseNames(obj)
                 for testname in testnames:
                     suite.addTest(obj(methodName=testname,
                                   filepath=filepath,
                                   testvars=testvars,
                                   **kwargs))
 
     def setUp(self):
-        Common2TestCase.setUp(self)
+        CommonTestCase.setUp(self)
 
     def tearDown(self):
-        Common2TestCase.tearDown(self)
+        CommonTestCase.tearDown(self)
 
     def wait_for_condition(self, method, timeout=30):
         timeout = float(timeout) + time.time()
         while time.time() < timeout:
             value = method(self.marionette)
             if value:
                 return value
             time.sleep(0.5)
         else:
             raise TimeoutException("wait_for_condition timed out")
 
-class Marionette2JSTestCase(Common2TestCase):
+class SessionJSTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.js$")
 
     def __init__(self, methodName='runTest', jsFile=None, **kwargs):
         assert(jsFile)
         self.jsFile = jsFile
         self.marionette = None
-        Common2TestCase.__init__(self, methodName)
+        CommonTestCase.__init__(self, methodName)
 
     @classmethod
     def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, testvars, **kwargs):
         suite.addTest(cls(jsFile=filepath, **kwargs))
 
     def runTest(self):
         self.run_js_test(self.jsFile)
 
rename from testing/marionette/harness/session/tests/test_marionette2.py
rename to testing/marionette/harness/session/tests/test_session.py
--- a/testing/marionette/harness/session/tests/unit-tests.ini
+++ b/testing/marionette/harness/session/tests/unit-tests.ini
@@ -6,9 +6,9 @@ qemu = false
 browser = true
 
 ; true if the test is compatible with b2g, otherwise false
 b2g = false
 
 ; true if the test should be skipped
 skip = false
 
-[test_marionette2.py]
+[test_session.py]
--- a/testing/marionette/mach_commands.py
+++ b/testing/marionette/mach_commands.py
@@ -23,17 +23,17 @@ MARIONETTE_DISABLED_B2G = '''
 The %s command requires a Marionette-enabled build.
 
 Please create an engineering build, which has Marionette enabled.  You can do
 this by ommitting the VARIANT variable when building, or using:
 
 VARIANT=eng ./build.sh
 '''
 
-def setup_argument_parser():
+def setup_marionette_argument_parser():
     from marionette.runner.base import BaseMarionetteArguments
     return BaseMarionetteArguments()
 
 def run_marionette(tests, b2g_path=None, emulator=None, testtype=None,
     address=None, binary=None, topsrcdir=None, **kwargs):
     from mozlog.structured import commandline
 
     from marionette.runtests import (
@@ -68,51 +68,51 @@ def run_marionette(tests, b2g_path=None,
                                             args,
                                             {"mach": sys.stdout})
     failed = MarionetteHarness(MarionetteTestRunner, args=args).run()
     if failed > 0:
         return 1
     else:
         return 0
 
-def setup_argument_parser2():
-    from marionette.runner.base2 import BaseMarionette2Arguments
-    return BaseMarionette2Arguments()
+def setup_session_argument_parser():
+    from session.runner.base import BaseSessionArguments
+    return BaseSessionArguments()
 
-def run_marionette2(tests, testtype=None,
+def run_session(tests, testtype=None,
     binary=None, topsrcdir=None, **kwargs):
     from mozlog.structured import commandline
 
-    from marionette.runtests2 import (
-        Marionette2TestRunner,
-        BaseMarionette2Arguments,
-        Marionette2Harness
+    from session.runtests import (
+        SessionTestRunner,
+        BaseSessionArguments,
+        SessionHarness
     )
 
-    parser = BaseMarionette2Arguments()
+    parser = BaseSessionArguments()
     commandline.add_logging_group(parser)
     args = parser.parse_args()
 
     if not tests:
         tests = [os.path.join(topsrcdir,
-                 'testing/marionette/harness/marionette/tests2/unit-tests.ini')]
+                 'testing/marionette/harness/session/tests/unit-tests.ini')]
     args.tests = tests
 
     args.binary = binary
     path, exe = os.path.split(args.binary)
 
     for k, v in kwargs.iteritems():
         setattr(args, k, v)
 
     parser.verify_usage(args)
 
-    args.logger = commandline.setup_logging("Marionette Unit Tests",
+    args.logger = commandline.setup_logging("Session Unit Tests",
                                             args,
                                             {"mach": sys.stdout})
-    failed = Marionette2Harness(Marionette2TestRunner, args=args).run()
+    failed = SessionHarness(SessionTestRunner, args=args).run()
     if failed > 0:
         return 1
     else:
         return 0
 
 @CommandProvider
 class B2GCommands(MachCommandBase):
     def __init__(self, context):
@@ -147,34 +147,34 @@ class B2GCommands(MachCommandBase):
         return run_marionette(tests, b2g_path=self.b2g_home, emulator=emulator,
             topsrcdir=self.topsrcdir, **kwargs)
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('marionette-test', category='testing',
         description='Run a Marionette test (Check UI or the internal JavaScript using marionette).',
         conditions=[conditions.is_firefox],
-        parser=setup_argument_parser,
+        parser=setup_marionette_argument_parser,
     )
     def run_marionette_test(self, tests, **kwargs):
         if 'test_objects' in kwargs:
             tests = []
             for obj in kwargs['test_objects']:
                 tests.append(obj['file_relpath'])
             del kwargs['test_objects']
 
         kwargs['binary'] = self.get_binary_path('app')
         return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)
 
-    @Command('marionette-session-test', category='testing',
-        description='Run a Marionette-Session test (Check Telemetry using marionette).',
+    @Command('session-test', category='testing',
+        description='Run a Session test (Check Telemetry using marionette).',
         conditions=[conditions.is_firefox],
-        parser=setup_argument_parser2,
+        parser=setup_session_argument_parser,
     )
-    def run_marionette2_test(self, tests, **kwargs):
+    def run_session_test(self, tests, **kwargs):
         if 'test_objects' in kwargs:
             tests = []
             for obj in kwargs['test_objects']:
                 tests.append(obj['file_relpath'])
             del kwargs['test_objects']
 
         kwargs['binary'] = self.get_binary_path('app')
-        return run_marionette2(tests, topsrcdir=self.topsrcdir, **kwargs)
+        return run_session(tests, topsrcdir=self.topsrcdir, **kwargs)