Bug 1340551 - Log tests by manifest from suite_start in marionette-harness, r?jgraham
MozReview-Commit-ID: 95XZAO4CfWP
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -8,16 +8,17 @@ import random
import re
import socket
import sys
import time
import traceback
import unittest
from argparse import ArgumentParser
+from collections import defaultdict
from copy import deepcopy
import mozinfo
import moznetwork
import mozprofile
import mozversion
import serve
@@ -839,17 +840,21 @@ class BaseMarionetteTestRunner(object):
appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False)
self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled"))
if self.e10s != appinfo_e10s:
message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does "
"not match browser appinfo")
self.cleanup()
raise AssertionError(message_e10s)
- self.logger.suite_start(self.tests,
+ tests_by_group = defaultdict(list)
+ for test in self.tests:
+ tests_by_group[test['group']].append(test['filepath'])
+
+ self.logger.suite_start(tests_by_group,
version_info=self.version_info,
device_info=device_info)
self._log_skipped_tests()
interrupted = None
try:
counter = self.repeat
@@ -913,17 +918,17 @@ class BaseMarionetteTestRunner(object):
def start_fixture_servers(self):
root = self.server_root or os.path.join(os.path.dirname(here), "www")
if self.appName == "fennec":
return serve.start(root, host=moznetwork.get_ip())
else:
return serve.start(root)
- def add_test(self, test, expected='pass'):
+ def add_test(self, test, expected='pass', group='default'):
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if filename.endswith('.ini'):
msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
" See --help for details.")
@@ -932,16 +937,18 @@ class BaseMarionetteTestRunner(object):
elif self._is_filename_valid(filename):
test_file = os.path.join(root, filename)
self.add_test(test_file)
return
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
if file_ext == '.ini':
+ group = filepath
+
manifest = TestManifest()
manifest.read(filepath)
json_path = update_mozinfo(filepath)
self.logger.info("mozinfo updated from: {}".format(json_path))
self.logger.info("mozinfo is: {}".format(mozinfo.info))
filters = []
@@ -970,22 +977,20 @@ class BaseMarionetteTestRunner(object):
self.manifest_skipped_tests.append(test)
else:
target_tests.append(test)
for i in target_tests:
if not os.path.exists(i["path"]):
raise IOError("test file: {} does not exist".format(i["path"]))
- file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
-
- self.add_test(i["path"], i["expected"])
+ self.add_test(i["path"], i["expected"], group=group)
return
- self.tests.append({'filepath': filepath, 'expected': expected})
+ self.tests.append({'filepath': filepath, 'expected': expected, 'group': group})
def run_test(self, filepath, expected):
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
self.test_kwargs['expected'] = expected
mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
for handler in self.test_handlers:
if handler.match(os.path.basename(filepath)):
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -268,17 +268,17 @@ def test_record_crash(runner, has_crashe
def test_add_test_module(runner):
tests = ['test_something.py', 'testSomething.js', 'bad_test.py']
assert len(runner.tests) == 0
for test in tests:
with patch('os.path.abspath', return_value=test) as abspath:
runner.add_test(test)
assert abspath.called
- expected = {'filepath': test, 'expected': 'pass'}
+ expected = {'filepath': test, 'expected': 'pass', 'group': 'default'}
assert expected in runner.tests
# add_test doesn't validate module names; 'bad_test.py' gets through
assert len(runner.tests) == 3
def test_add_test_directory(runner):
test_dir = 'path/to/tests'
dir_contents = [