Bug 1340551 - [mozlog] Change 'tests' field in suite_start action to a dict of tests keyed by group name, r?jgraham
In suite_start, tests is currently a list of test ids. But to support a new 'test-centric' treeherder view, we need to
be able to map tests to their manifests somewhere in the structured log, and the suite_start tests field seemed like a
good spot.
Because not all test suites use manifests, this is being called a "group" so it could potentially be re-used with
directories, tags or subsuites.
MozReview-Commit-ID: 2Sc7nqJrWts
--- a/testing/mozbase/docs/mozlog.rst
+++ b/testing/mozbase/docs/mozlog.rst
@@ -58,21 +58,25 @@ on on all messages is:
For each ``action`` there are is a further set of specific fields
describing the details of the event that caused the message to be
emitted:
``suite_start``
Emitted when the testsuite starts running.
``tests``
- A list of test ids. Test ids can either be strings or lists of
- strings (an example of the latter is reftests where the id has the
- form [test_url, ref_type, ref_url]) and are assumed to be unique
- within a given testsuite. In cases where the test list is not
- known upfront an empty list may be passed (list).
+ A dict of test ids keyed by group. Groups are any logical grouping
+ of tests, for example a manifest, directory or tag. For convenience,
+ a list of test ids can be used instead. In this case all tests will
+ automatically be placed in the 'default' group name. Test ids can
+ either be strings or lists of strings (an example of the latter is
+ reftests where the id has the form [test_url, ref_type, ref_url]).
+ Test ids are assumed to be unique within a given testsuite. In cases
+ where the test list is not known upfront an empty dict or list may
+ be passed (dict).
``run_info``
An optional dictionary describing the properties of the
build and test environment. This contains the information provided
by :doc:`mozinfo <mozinfo>`, plus a boolean ``debug`` field indicating
whether the build under test is a debug build.
``suite_end``
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -116,17 +116,18 @@ class MachFormatter(base.BaseFormatter):
def suite_start(self, data):
self.summary_values = {"tests": 0,
"subtests": 0,
"assertion_counts": 0,
"expected": 0,
"unexpected": defaultdict(int),
"skipped": 0}
self.summary_unexpected = []
- return "%i" % len(data["tests"])
+ num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+ return "%i" % num_tests
def suite_end(self, data):
term = self.terminal if self.terminal is not None else NullTerminal()
heading = "Summary"
rv = ["", heading, "=" * len(heading), ""]
has_subtests = self.summary_values["subtests"] > 0
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -112,17 +112,18 @@ class TbplFormatter(BaseFormatter):
rv = "\n".join(rv)
if not rv[-1] == "\n":
rv += "\n"
return rv
def suite_start(self, data):
self.suite_start_time = data["time"]
- return "SUITE-START | Running %i tests\n" % len(data["tests"])
+ num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+ return "SUITE-START | Running %i tests\n" % num_tests
def test_start(self, data):
self.test_start_times[self.test_id(data["test"])] = data["time"]
return "TEST-START | %s\n" % data["test"]
def test_status(self, data):
if self.compact:
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -205,22 +205,34 @@ class Dict(ContainerType):
def convert(self, data):
key_type, value_type = self.item_type
return {key_type.convert(k): value_type.convert(v) for k, v in dict(data).items()}
class List(ContainerType):
def convert(self, data):
- # while dicts and strings _can_ be cast to lists, doing so is probably not intentional
+ # while dicts and strings _can_ be cast to lists,
+ # doing so is likely not intentional behaviour
if isinstance(data, (basestring, dict)):
raise ValueError("Expected list but got %s" % type(data))
return [self.item_type.convert(item) for item in data]
+class TestList(DataType):
+ """A TestList is a list of tests that can be either keyed by a group name,
+ or specified as a flat list.
+ """
+
+ def convert(self, data):
+ if isinstance(data, (list, tuple)):
+ data = {'default': data}
+ return Dict({Unicode: List(Unicode)}).convert(data)
+
+
class Int(DataType):
def convert(self, data):
return int(data)
class Any(DataType):
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -6,17 +6,17 @@ from __future__ import unicode_literals
from multiprocessing import current_process
from threading import current_thread, Lock
import json
import sys
import time
import traceback
-from logtypes import Unicode, TestId, Status, SubStatus, Dict, List, Int, Any, Tuple
+from logtypes import Unicode, TestId, TestList, Status, SubStatus, Dict, List, Int, Any, Tuple
from logtypes import log_action, convertor_registry
"""Structured Logging for recording test results.
Allowed actions, and subfields:
suite_start
tests - List of test names
@@ -251,25 +251,25 @@ class StructuredLogger(object):
elif action == 'suite_end':
if not self._state.suite_started:
self.error("Got suite_end message before suite_start. " +
"Logged with data: {}".format(json.dumps(data)))
return False
self._state.suite_started = False
return True
- @log_action(List(Unicode, "tests"),
+ @log_action(TestList("tests"),
Dict(Any, "run_info", default=None, optional=True),
Dict(Any, "version_info", default=None, optional=True),
Dict(Any, "device_info", default=None, optional=True),
Dict(Any, "extra", default=None, optional=True))
def suite_start(self, data):
"""Log a suite_start message
- :param list tests: Test identifiers that will be run in the suite.
+ :param dict tests: Test identifiers that will be run in the suite, keyed by group name.
:param dict run_info: Optional information typically provided by mozinfo.
:param dict version_info: Optional target application version information provided
by mozversion.
:param dict device_info: Optional target device information provided by mozdevice.
"""
if not self._ensure_suite_state('suite_start', data):
return
--- a/testing/mozbase/mozlog/tests/test_logtypes.py
+++ b/testing/mozbase/mozlog/tests/test_logtypes.py
@@ -5,16 +5,17 @@
import unittest
import mozunit
from mozlog.logtypes import (
Any,
Dict,
Int,
List,
+ TestList,
Tuple,
Unicode,
)
class TestContainerTypes(unittest.TestCase):
def test_dict_type_basic(self):
@@ -90,10 +91,26 @@ class TestContainerTypes(unittest.TestCa
t(({'foo': 'bar'}, [{'foo': 'bar'}], 'foo'))
with self.assertRaises(ValueError):
t(({'foo': ['bar']}, ['foo'], 'foo'))
t(({'foo': ['bar']}, [{'foo': 'bar'}], 'foo')) # doesn't raise
+class TestDataTypes(unittest.TestCase):
+
+ def test_test_list(self):
+ t = TestList('name')
+ with self.assertRaises(ValueError):
+ t('foo')
+
+ with self.assertRaises(ValueError):
+ t({'foo': 1})
+
+ d1 = t({'default': ['bar']}) # doesn't raise
+ d2 = t(['bar']) # doesn't raise
+
+ self.assertDictContainsSubset(d1, d2)
+
+
if __name__ == '__main__':
mozunit.main()
--- a/testing/mozbase/mozlog/tests/test_structured.py
+++ b/testing/mozbase/mozlog/tests/test_structured.py
@@ -105,17 +105,17 @@ class TestStatusHandler(BaseStructuredTe
self.assertEqual(2, summary.expected_statuses['OK'])
class TestStructuredLog(BaseStructuredTest):
def test_suite_start(self):
self.logger.suite_start(["test"])
self.assert_log_equals({"action": "suite_start",
- "tests": ["test"]})
+ "tests": {"default": ["test"]}})
self.logger.suite_end()
def test_suite_end(self):
self.logger.suite_start([])
self.logger.suite_end()
self.assert_log_equals({"action": "suite_end"})
def test_start(self):
@@ -258,27 +258,27 @@ class TestStructuredLog(BaseStructuredTe
self.assertEquals(last_item["level"], "ERROR")
self.assertTrue(last_item["message"].startswith(
"test_end for test2 logged while not in progress. Logged with data: {"))
self.logger.suite_end()
def test_suite_start_twice(self):
self.logger.suite_start([])
self.assert_log_equals({"action": "suite_start",
- "tests": []})
+ "tests": {"default": []}})
self.logger.suite_start([])
last_item = self.pop_last_item()
self.assertEquals(last_item["action"], "log")
self.assertEquals(last_item["level"], "ERROR")
self.logger.suite_end()
def test_suite_end_no_start(self):
self.logger.suite_start([])
self.assert_log_equals({"action": "suite_start",
- "tests": []})
+ "tests": {"default": []}})
self.logger.suite_end()
self.assert_log_equals({"action": "suite_end"})
self.logger.suite_end()
last_item = self.pop_last_item()
self.assertEquals(last_item["action"], "log")
self.assertEquals(last_item["level"], "ERROR")
def test_multiple_loggers_suite_start(self):
@@ -396,17 +396,17 @@ class TestStructuredLog(BaseStructuredTe
class TestTypeConversions(BaseStructuredTest):
def test_raw(self):
self.logger.log_raw({"action": "suite_start",
"tests": [1],
"time": "1234"})
self.assert_log_equals({"action": "suite_start",
- "tests": ["1"],
+ "tests": {"default": ["1"]},
"time": 1234})
self.logger.suite_end()
def test_tuple(self):
self.logger.suite_start([])
self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90\x8d\x83\xf0\x90\x8d\x84",
42, u"\u16a4"))
self.assert_log_equals({"action": "test_start",
@@ -441,17 +441,17 @@ class TestTypeConversions(BaseStructured
def test_arguments(self):
self.logger.info(message="test")
self.assert_log_equals({"action": "log",
"message": "test",
"level": "INFO"})
self.logger.suite_start([], {})
self.assert_log_equals({"action": "suite_start",
- "tests": [],
+ "tests": {"default": []},
"run_info": {}})
self.logger.test_start(test="test1")
self.logger.test_status(
"subtest1", "FAIL", test="test1", status="PASS")
self.assert_log_equals({"action": "test_status",
"test": "test1",
"subtest": "subtest1",
"status": "PASS",
@@ -1007,17 +1007,17 @@ class TestBuffer(BaseStructuredTest):
"test": "test1",
"status": "PASS",
"subtest": "sub6"})
self.assert_log_equals({"action": "test_status",
"test": "test1",
"status": "PASS",
"subtest": "sub5"})
self.assert_log_equals({"action": "suite_start",
- "tests": []})
+ "tests": {"default": []}})
class TestReader(unittest.TestCase):
def to_file_like(self, obj):
data_str = "\n".join(json.dumps(item) for item in obj)
return StringIO.StringIO(data_str)