Bug 1340551 - Log tests by manifest from suite_start in marionette-harness, r?jgraham draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 22 Feb 2017 15:09:02 -0500
changeset 490232 be86a51161c9a891ed26cb0bf19eabf0cd378864
parent 490231 94c5ffb6252978ce421d1c1503b34ceafa810d42
child 490233 4a4f4940e1cfff3ac35212491677fdce5d735d99
push id47032
push userahalberstadt@mozilla.com
push dateMon, 27 Feb 2017 21:18:22 +0000
reviewersjgraham
bugs1340551
milestone54.0a1
Bug 1340551 - Log tests by manifest from suite_start in marionette-harness, r?jgraham MozReview-Commit-ID: 95XZAO4CfWP
testing/marionette/harness/marionette_harness/runner/base.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -8,16 +8,17 @@ import random
 import re
 import socket
 import sys
 import time
 import traceback
 import unittest
 
 from argparse import ArgumentParser
+from collections import defaultdict
 from copy import deepcopy
 
 import mozinfo
 import moznetwork
 import mozprofile
 import mozversion
 import serve
 
@@ -839,17 +840,21 @@ class BaseMarionetteTestRunner(object):
         appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False)
         self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled"))
         if self.e10s != appinfo_e10s:
             message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does "
                             "not match browser appinfo")
             self.cleanup()
             raise AssertionError(message_e10s)
 
-        self.logger.suite_start(self.tests,
+        tests_by_group = defaultdict(list)
+        for test in self.tests:
+            tests_by_group[test['group']].append(test['filepath'])
+
+        self.logger.suite_start(tests_by_group,
                                 version_info=self.version_info,
                                 device_info=device_info)
 
         self._log_skipped_tests()
 
         interrupted = None
         try:
             counter = self.repeat
@@ -913,17 +918,17 @@ class BaseMarionetteTestRunner(object):
 
     def start_fixture_servers(self):
         root = self.server_root or os.path.join(os.path.dirname(here), "www")
         if self.appName == "fennec":
             return serve.start(root, host=moznetwork.get_ip())
         else:
             return serve.start(root)
 
-    def add_test(self, test, expected='pass'):
+    def add_test(self, test, expected='pass', group='default'):
         filepath = os.path.abspath(test)
 
         if os.path.isdir(filepath):
             for root, dirs, files in os.walk(filepath):
                 for filename in files:
                     if filename.endswith('.ini'):
                         msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
                                     " See --help for details.")
@@ -932,16 +937,18 @@ class BaseMarionetteTestRunner(object):
                     elif self._is_filename_valid(filename):
                         test_file = os.path.join(root, filename)
                         self.add_test(test_file)
             return
 
         file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
 
         if file_ext == '.ini':
+            group = filepath
+
             manifest = TestManifest()
             manifest.read(filepath)
 
             json_path = update_mozinfo(filepath)
             self.logger.info("mozinfo updated from: {}".format(json_path))
             self.logger.info("mozinfo is: {}".format(mozinfo.info))
 
             filters = []
@@ -970,22 +977,20 @@ class BaseMarionetteTestRunner(object):
                     self.manifest_skipped_tests.append(test)
                 else:
                     target_tests.append(test)
 
             for i in target_tests:
                 if not os.path.exists(i["path"]):
                     raise IOError("test file: {} does not exist".format(i["path"]))
 
-                file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
-
-                self.add_test(i["path"], i["expected"])
+                self.add_test(i["path"], i["expected"], group=group)
             return
 
-        self.tests.append({'filepath': filepath, 'expected': expected})
+        self.tests.append({'filepath': filepath, 'expected': expected, 'group': group})
 
     def run_test(self, filepath, expected):
         testloader = unittest.TestLoader()
         suite = unittest.TestSuite()
         self.test_kwargs['expected'] = expected
         mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
         for handler in self.test_handlers:
             if handler.match(os.path.basename(filepath)):
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -268,17 +268,17 @@ def test_record_crash(runner, has_crashe
 
 def test_add_test_module(runner):
     tests = ['test_something.py', 'testSomething.js', 'bad_test.py']
     assert len(runner.tests) == 0
     for test in tests:
         with patch('os.path.abspath', return_value=test) as abspath:
             runner.add_test(test)
         assert abspath.called
-        expected = {'filepath': test, 'expected': 'pass'}
+        expected = {'filepath': test, 'expected': 'pass', 'group': 'default'}
         assert expected in runner.tests
     # add_test doesn't validate module names; 'bad_test.py' gets through
     assert len(runner.tests) == 3
 
 
 def test_add_test_directory(runner):
     test_dir = 'path/to/tests'
     dir_contents = [