Bug 1340551 - [mozlog] Change 'tests' field in suite_start action to a dict of tests keyed by group name, r?jgraham draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 22 Feb 2017 11:46:09 -0500
changeset 490229 299fbbb5aefc57e943c3d4cc2ac9f6077ba79690
parent 490105 a5289d7a3e56cab836772c33c791bff99b24e674
child 490230 668dbd323778126d4e80c9a117dbea42933f9ffa
push id47032
push userahalberstadt@mozilla.com
push dateMon, 27 Feb 2017 21:18:22 +0000
reviewersjgraham
bugs1340551
milestone54.0a1
Bug 1340551 - [mozlog] Change 'tests' field in suite_start action to a dict of tests keyed by group name, r?jgraham In suite_start, tests is currently a list of test ids. But to support a new 'test-centric' treeherder view, we need to be able to map tests to their manifests somewhere in the structured log, and the suite_start tests field seemed like a good spot. Because not all test suites use manifests, this is being called a "group" so it could potentially be re-used with directories, tags or subsuites. MozReview-Commit-ID: 2Sc7nqJrWts
testing/mozbase/docs/mozlog.rst
testing/mozbase/mozlog/mozlog/formatters/machformatter.py
testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
testing/mozbase/mozlog/mozlog/logtypes.py
testing/mozbase/mozlog/mozlog/structuredlog.py
testing/mozbase/mozlog/tests/test_logtypes.py
testing/mozbase/mozlog/tests/test_structured.py
--- a/testing/mozbase/docs/mozlog.rst
+++ b/testing/mozbase/docs/mozlog.rst
@@ -58,21 +58,25 @@ on on all messages is:
 For each ``action`` there are is a further set of specific fields
 describing the details of the event that caused the message to be
 emitted:
 
 ``suite_start``
   Emitted when the testsuite starts running.
 
   ``tests``
-    A list of test ids. Test ids can either be strings or lists of
-    strings (an example of the latter is reftests where the id has the
-    form [test_url, ref_type, ref_url]) and are assumed to be unique
-    within a given testsuite. In cases where the test list is not
-    known upfront an empty list may be passed (list).
+    A dict of test ids keyed by group. Groups are any logical grouping
+    of tests, for example a manifest, directory or tag. For convenience,
+    a list of test ids can be used instead. In this case all tests will
+    automatically be placed in the 'default' group name. Test ids can
+    either be strings or lists of strings (an example of the latter is
+    reftests where the id has the form [test_url, ref_type, ref_url]).
+    Test ids are assumed to be unique within a given testsuite. In cases
+    where the test list is not known upfront an empty dict or list may
+    be passed (dict).
 
   ``run_info``
     An optional dictionary describing the properties of the
     build and test environment. This contains the information provided
     by :doc:`mozinfo <mozinfo>`, plus a boolean ``debug`` field indicating
     whether the build under test is a debug build.
 
 ``suite_end``
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -116,17 +116,18 @@ class MachFormatter(base.BaseFormatter):
     def suite_start(self, data):
         self.summary_values = {"tests": 0,
                                "subtests": 0,
                                "assertion_counts": 0,
                                "expected": 0,
                                "unexpected": defaultdict(int),
                                "skipped": 0}
         self.summary_unexpected = []
-        return "%i" % len(data["tests"])
+        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        return "%i" % num_tests
 
     def suite_end(self, data):
         term = self.terminal if self.terminal is not None else NullTerminal()
 
         heading = "Summary"
         rv = ["", heading, "=" * len(heading), ""]
 
         has_subtests = self.summary_values["subtests"] > 0
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -112,17 +112,18 @@ class TbplFormatter(BaseFormatter):
         rv = "\n".join(rv)
         if not rv[-1] == "\n":
             rv += "\n"
 
         return rv
 
     def suite_start(self, data):
         self.suite_start_time = data["time"]
-        return "SUITE-START | Running %i tests\n" % len(data["tests"])
+        num_tests = reduce(lambda x, y: x + len(y), data['tests'].itervalues(), 0)
+        return "SUITE-START | Running %i tests\n" % num_tests
 
     def test_start(self, data):
         self.test_start_times[self.test_id(data["test"])] = data["time"]
 
         return "TEST-START | %s\n" % data["test"]
 
     def test_status(self, data):
         if self.compact:
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -205,22 +205,34 @@ class Dict(ContainerType):
     def convert(self, data):
         key_type, value_type = self.item_type
         return {key_type.convert(k): value_type.convert(v) for k, v in dict(data).items()}
 
 
 class List(ContainerType):
 
     def convert(self, data):
-        # while dicts and strings _can_ be cast to lists, doing so is probably not intentional
+        # while dicts and strings _can_ be cast to lists,
+        # doing so is likely not intentional behaviour
         if isinstance(data, (basestring, dict)):
             raise ValueError("Expected list but got %s" % type(data))
         return [self.item_type.convert(item) for item in data]
 
 
+class TestList(DataType):
+    """A TestList is a list of tests that can be either keyed by a group name,
+    or specified as a flat list.
+    """
+
+    def convert(self, data):
+        if isinstance(data, (list, tuple)):
+            data = {'default': data}
+        return Dict({Unicode: List(Unicode)}).convert(data)
+
+
 class Int(DataType):
 
     def convert(self, data):
         return int(data)
 
 
 class Any(DataType):
 
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -6,17 +6,17 @@ from __future__ import unicode_literals
 
 from multiprocessing import current_process
 from threading import current_thread, Lock
 import json
 import sys
 import time
 import traceback
 
-from logtypes import Unicode, TestId, Status, SubStatus, Dict, List, Int, Any, Tuple
+from logtypes import Unicode, TestId, TestList, Status, SubStatus, Dict, List, Int, Any, Tuple
 from logtypes import log_action, convertor_registry
 
 """Structured Logging for recording test results.
 
 Allowed actions, and subfields:
   suite_start
       tests  - List of test names
 
@@ -251,25 +251,25 @@ class StructuredLogger(object):
         elif action == 'suite_end':
             if not self._state.suite_started:
                 self.error("Got suite_end message before suite_start. " +
                            "Logged with data: {}".format(json.dumps(data)))
                 return False
             self._state.suite_started = False
         return True
 
-    @log_action(List(Unicode, "tests"),
+    @log_action(TestList("tests"),
                 Dict(Any, "run_info", default=None, optional=True),
                 Dict(Any, "version_info", default=None, optional=True),
                 Dict(Any, "device_info", default=None, optional=True),
                 Dict(Any, "extra", default=None, optional=True))
     def suite_start(self, data):
         """Log a suite_start message
 
-        :param list tests: Test identifiers that will be run in the suite.
+        :param dict tests: Test identifiers that will be run in the suite, keyed by group name.
         :param dict run_info: Optional information typically provided by mozinfo.
         :param dict version_info: Optional target application version information provided
           by mozversion.
         :param dict device_info: Optional target device information provided by mozdevice.
         """
         if not self._ensure_suite_state('suite_start', data):
             return
 
--- a/testing/mozbase/mozlog/tests/test_logtypes.py
+++ b/testing/mozbase/mozlog/tests/test_logtypes.py
@@ -5,16 +5,17 @@
 import unittest
 import mozunit
 
 from mozlog.logtypes import (
     Any,
     Dict,
     Int,
     List,
+    TestList,
     Tuple,
     Unicode,
 )
 
 
 class TestContainerTypes(unittest.TestCase):
 
     def test_dict_type_basic(self):
@@ -90,10 +91,26 @@ class TestContainerTypes(unittest.TestCa
             t(({'foo': 'bar'}, [{'foo': 'bar'}], 'foo'))
 
         with self.assertRaises(ValueError):
             t(({'foo': ['bar']}, ['foo'], 'foo'))
 
         t(({'foo': ['bar']}, [{'foo': 'bar'}], 'foo'))  # doesn't raise
 
 
+class TestDataTypes(unittest.TestCase):
+
+    def test_test_list(self):
+        t = TestList('name')
+        with self.assertRaises(ValueError):
+            t('foo')
+
+        with self.assertRaises(ValueError):
+            t({'foo': 1})
+
+        d1 = t({'default': ['bar']})  # doesn't raise
+        d2 = t(['bar'])  # doesn't raise
+
+        self.assertDictContainsSubset(d1, d2)
+
+
 if __name__ == '__main__':
     mozunit.main()
--- a/testing/mozbase/mozlog/tests/test_structured.py
+++ b/testing/mozbase/mozlog/tests/test_structured.py
@@ -105,17 +105,17 @@ class TestStatusHandler(BaseStructuredTe
         self.assertEqual(2, summary.expected_statuses['OK'])
 
 
 class TestStructuredLog(BaseStructuredTest):
 
     def test_suite_start(self):
         self.logger.suite_start(["test"])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": ["test"]})
+                                "tests": {"default": ["test"]}})
         self.logger.suite_end()
 
     def test_suite_end(self):
         self.logger.suite_start([])
         self.logger.suite_end()
         self.assert_log_equals({"action": "suite_end"})
 
     def test_start(self):
@@ -258,27 +258,27 @@ class TestStructuredLog(BaseStructuredTe
         self.assertEquals(last_item["level"], "ERROR")
         self.assertTrue(last_item["message"].startswith(
             "test_end for test2 logged while not in progress. Logged with data: {"))
         self.logger.suite_end()
 
     def test_suite_start_twice(self):
         self.logger.suite_start([])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
         self.logger.suite_start([])
         last_item = self.pop_last_item()
         self.assertEquals(last_item["action"], "log")
         self.assertEquals(last_item["level"], "ERROR")
         self.logger.suite_end()
 
     def test_suite_end_no_start(self):
         self.logger.suite_start([])
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
         self.logger.suite_end()
         self.assert_log_equals({"action": "suite_end"})
         self.logger.suite_end()
         last_item = self.pop_last_item()
         self.assertEquals(last_item["action"], "log")
         self.assertEquals(last_item["level"], "ERROR")
 
     def test_multiple_loggers_suite_start(self):
@@ -396,17 +396,17 @@ class TestStructuredLog(BaseStructuredTe
 
 class TestTypeConversions(BaseStructuredTest):
 
     def test_raw(self):
         self.logger.log_raw({"action": "suite_start",
                              "tests": [1],
                              "time": "1234"})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": ["1"],
+                                "tests": {"default": ["1"]},
                                 "time": 1234})
         self.logger.suite_end()
 
     def test_tuple(self):
         self.logger.suite_start([])
         self.logger.test_start(("\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90\x8d\x83\xf0\x90\x8d\x84",
                                 42, u"\u16a4"))
         self.assert_log_equals({"action": "test_start",
@@ -441,17 +441,17 @@ class TestTypeConversions(BaseStructured
     def test_arguments(self):
         self.logger.info(message="test")
         self.assert_log_equals({"action": "log",
                                 "message": "test",
                                 "level": "INFO"})
 
         self.logger.suite_start([], {})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": [],
+                                "tests": {"default": []},
                                 "run_info": {}})
         self.logger.test_start(test="test1")
         self.logger.test_status(
             "subtest1", "FAIL", test="test1", status="PASS")
         self.assert_log_equals({"action": "test_status",
                                 "test": "test1",
                                 "subtest": "subtest1",
                                 "status": "PASS",
@@ -1007,17 +1007,17 @@ class TestBuffer(BaseStructuredTest):
                                 "test": "test1",
                                 "status": "PASS",
                                 "subtest": "sub6"})
         self.assert_log_equals({"action": "test_status",
                                 "test": "test1",
                                 "status": "PASS",
                                 "subtest": "sub5"})
         self.assert_log_equals({"action": "suite_start",
-                                "tests": []})
+                                "tests": {"default": []}})
 
 
 class TestReader(unittest.TestCase):
 
     def to_file_like(self, obj):
         data_str = "\n".join(json.dumps(item) for item in obj)
         return StringIO.StringIO(data_str)