Bug 1340551 - [mozlog] Introduce concept of ContainerType in logtypes and allow nested containers, r?jgraham draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 21 Feb 2017 14:24:14 -0500
changeset 490105 a5289d7a3e56cab836772c33c791bff99b24e674
parent 489990 ecf39ae188da37206b39a31e585fdcb028bd8b7b
child 490106 d7c56f1ac686d31a4cb059433608481cb054fa73
child 490229 299fbbb5aefc57e943c3d4cc2ac9f6077ba79690
push id46996
push userahalberstadt@mozilla.com
push dateMon, 27 Feb 2017 17:20:18 +0000
reviewersjgraham
bugs1340551
milestone54.0a1
Bug 1340551 - [mozlog] Introduce concept of ContainerType in logtypes and allow nested containers, r?jgraham Currently the List and Tuple DataTypes must specify what items they contain. But there's no way to specify item types recursively, e.g List(Tuple(Int, Int)). Also the Dict type can't specify the item types it must contain either. Dict is a special case because we may want to control both keys and values. This patch formalizes a ContainerType (of which List, Tuple and Dict are subclasses). MozReview-Commit-ID: Bouhy1DIAyD
testing/mozbase/mozlog/mozlog/logtypes.py
testing/mozbase/mozlog/mozlog/structuredlog.py
testing/mozbase/mozlog/tests/manifest.ini
testing/mozbase/mozlog/tests/test_logtypes.py
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -1,12 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import inspect
+
 convertor_registry = {}
 missing = object()
 no_default = object()
 
 
 class log_action(object):
 
     def __init__(self, *args):
@@ -118,16 +120,41 @@ class DataType(object):
 
         try:
             return self.convert(value)
         except:
             raise ValueError("Failed to convert value %s of type %s for field %s to type %s" %
                              (value, type(value).__name__, self.name, self.__class__.__name__))
 
 
+class ContainerType(DataType):
+    """A DataType that contains other DataTypes.
+
+    ContainerTypes must specify which other DataType they will contain. ContainerTypes
+    may contain other ContainerTypes.
+
+    Some examples:
+
+        List(Int, 'numbers')
+        Tuple((Unicode, Int, Any), 'things')
+        Dict(Unicode, 'config')
+        Dict({TestId: Status}, 'results')
+        Dict(List(Unicode), 'stuff')
+    """
+
+    def __init__(self, item_type, name=None, **kwargs):
+        DataType.__init__(self, name, **kwargs)
+        self.item_type = self._format_item_type(item_type)
+
+    def _format_item_type(self, item_type):
+        if inspect.isclass(item_type):
+            return item_type(None)
+        return item_type
+
+
 class Unicode(DataType):
 
     def convert(self, data):
         if isinstance(data, unicode):
             return data
         if isinstance(data, str):
             return data.decode("utf8", "replace")
         return unicode(data)
@@ -158,47 +185,60 @@ class Status(DataType):
             raise ValueError
         return value
 
 
 class SubStatus(Status):
     allowed = ["PASS", "FAIL", "ERROR", "TIMEOUT", "ASSERT", "NOTRUN", "SKIP"]
 
 
-class Dict(DataType):
+class Dict(ContainerType):
+
+    def _format_item_type(self, item_type):
+        superfmt = super(Dict, self)._format_item_type
+
+        if isinstance(item_type, dict):
+            if len(item_type) != 1:
+                raise ValueError("Dict item type specifier must contain a single entry.")
+            key_type, value_type = item_type.items()[0]
+            return superfmt(key_type), superfmt(value_type)
+        return Any(None), superfmt(item_type)
 
     def convert(self, data):
-        return dict(data)
+        key_type, value_type = self.item_type
+        return {key_type.convert(k): value_type.convert(v) for k, v in dict(data).items()}
 
 
-class List(DataType):
-
-    def __init__(self, name, item_type, default=no_default, optional=False):
-        DataType.__init__(self, name, default, optional)
-        self.item_type = item_type(None)
+class List(ContainerType):
 
     def convert(self, data):
+        # while dicts and strings _can_ be cast to lists, doing so is probably not intentional
+        if isinstance(data, (basestring, dict)):
+            raise ValueError("Expected list but got %s" % type(data))
         return [self.item_type.convert(item) for item in data]
 
 
 class Int(DataType):
 
     def convert(self, data):
         return int(data)
 
 
 class Any(DataType):
 
     def convert(self, data):
         return data
 
 
-class Tuple(DataType):
+class Tuple(ContainerType):
 
-    def __init__(self, name, item_types, default=no_default, optional=False):
-        DataType.__init__(self, name, default, optional)
-        self.item_types = item_types
+    def _format_item_type(self, item_type):
+        superfmt = super(Tuple, self)._format_item_type
+
+        if isinstance(item_type, (tuple, list)):
+            return [superfmt(t) for t in item_type]
+        return (superfmt(item_type),)
 
     def convert(self, data):
-        if len(data) != len(self.item_types):
-            raise ValueError("Expected %i items got %i" % (len(self.item_types), len(data)))
+        if len(data) != len(self.item_type):
+            raise ValueError("Expected %i items got %i" % (len(self.item_type), len(data)))
         return tuple(item_type.convert(value)
-                     for item_type, value in zip(self.item_types, data))
+                     for item_type, value in zip(self.item_type, data))
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -251,36 +251,36 @@ class StructuredLogger(object):
         elif action == 'suite_end':
             if not self._state.suite_started:
                 self.error("Got suite_end message before suite_start. " +
                            "Logged with data: {}".format(json.dumps(data)))
                 return False
             self._state.suite_started = False
         return True
 
-    @log_action(List("tests", Unicode),
-                Dict("run_info", default=None, optional=True),
-                Dict("version_info", default=None, optional=True),
-                Dict("device_info", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+    @log_action(List(Unicode, "tests"),
+                Dict(Any, "run_info", default=None, optional=True),
+                Dict(Any, "version_info", default=None, optional=True),
+                Dict(Any, "device_info", default=None, optional=True),
+                Dict(Any, "extra", default=None, optional=True))
     def suite_start(self, data):
         """Log a suite_start message
 
         :param list tests: Test identifiers that will be run in the suite.
         :param dict run_info: Optional information typically provided by mozinfo.
         :param dict version_info: Optional target application version information provided
           by mozversion.
         :param dict device_info: Optional target device information provided by mozdevice.
         """
         if not self._ensure_suite_state('suite_start', data):
             return
 
         self._log_data("suite_start", data)
 
-    @log_action(Dict("extra", default=None, optional=True))
+    @log_action(Dict(Any, "extra", default=None, optional=True))
     def suite_end(self, data):
         """Log a suite_end message"""
         if not self._ensure_suite_state('suite_end', data):
             return
 
         self._log_data("suite_end")
 
     @log_action(TestId("test"),
@@ -304,17 +304,17 @@ class StructuredLogger(object):
         self._log_data("test_start", data)
 
     @log_action(TestId("test"),
                 Unicode("subtest"),
                 SubStatus("status"),
                 SubStatus("expected", default="PASS"),
                 Unicode("message", default=None, optional=True),
                 Unicode("stack", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+                Dict(Any, "extra", default=None, optional=True))
     def test_status(self, data):
         """
         Log a test_status message indicating a subtest result. Tests that
         do not have subtests are not expected to produce test_status messages.
 
         :param test: Identifier of the test that produced the result.
         :param subtest: Name of the subtest.
         :param status: Status string indicating the subtest result
@@ -335,17 +335,17 @@ class StructuredLogger(object):
 
         self._log_data("test_status", data)
 
     @log_action(TestId("test"),
                 Status("status"),
                 Status("expected", default="OK"),
                 Unicode("message", default=None, optional=True),
                 Unicode("stack", default=None, optional=True),
-                Dict("extra", default=None, optional=True))
+                Dict(Any, "extra", default=None, optional=True))
     def test_end(self, data):
         """
         Log a test_end message indicating that a test completed. For tests
         with subtests this indicates whether the overall test completed without
         errors. For tests without subtests this indicates the test result
         directly.
 
         :param test: Identifier of the test that produced the result.
@@ -384,25 +384,25 @@ class StructuredLogger(object):
     @log_action(Unicode("process", default=None),
                 Unicode("signature", default="[Unknown]"),
                 TestId("test", default=None, optional=True),
                 Unicode("minidump_path", default=None, optional=True),
                 Unicode("minidump_extra", default=None, optional=True),
                 Int("stackwalk_retcode", default=None, optional=True),
                 Unicode("stackwalk_stdout", default=None, optional=True),
                 Unicode("stackwalk_stderr", default=None, optional=True),
-                List("stackwalk_errors", Unicode, default=None))
+                List(Unicode, "stackwalk_errors", default=None))
     def crash(self, data):
         if data["stackwalk_errors"] is None:
             data["stackwalk_errors"] = []
 
         self._log_data("crash", data)
 
     @log_action(Unicode("primary", default=None),
-                List("secondary", Unicode, default=None))
+                List(Unicode, "secondary", default=None))
     def valgrind_error(self, data):
         self._log_data("valgrind_error", data)
 
     @log_action(Unicode("process"),
                 Unicode("command", default=None, optional=True))
     def process_start(self, data):
         """Log start event of a process.
 
@@ -471,17 +471,17 @@ def _log_func(level_name):
 def _lint_func(level_name):
     @log_action(Unicode("path"),
                 Unicode("message", default=""),
                 Int("lineno", default=0),
                 Int("column", default=None, optional=True),
                 Unicode("hint", default=None, optional=True),
                 Unicode("source", default=None, optional=True),
                 Unicode("rule", default=None, optional=True),
-                Tuple("lineoffset", (Int, Int), default=None, optional=True),
+                Tuple((Int, Int), "lineoffset", default=None, optional=True),
                 Unicode("linter", default=None, optional=True))
     def lint(self, data):
         data["level"] = level_name
         self._log_data("lint", data)
     lint.__doc__ = """Log an error resulting from a failed lint check
 
         :param linter: name of the linter that flagged this error
         :param path: path to the file containing the error
--- a/testing/mozbase/mozlog/tests/manifest.ini
+++ b/testing/mozbase/mozlog/tests/manifest.ini
@@ -1,4 +1,5 @@
 [DEFAULT]
 subsuite = mozbase, os == "linux"
 [test_logger.py]
+[test_logtypes.py]
 [test_structured.py]
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/mozlog/tests/test_logtypes.py
@@ -0,0 +1,99 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+
+from mozlog.logtypes import (
+    Any,
+    Dict,
+    Int,
+    List,
+    Tuple,
+    Unicode,
+)
+
+
+class TestContainerTypes(unittest.TestCase):
+
+    def test_dict_type_basic(self):
+        d = Dict('name')
+        with self.assertRaises(ValueError):
+            d({'foo': 'bar'})
+
+        d = Dict(Any, 'name')
+        d({'foo': 'bar'})  # doesn't raise
+
+    def test_dict_type_with_dictionary_item_type(self):
+        d = Dict({Int: Int}, 'name')
+        with self.assertRaises(ValueError):
+            d({'foo': 1})
+
+        with self.assertRaises(ValueError):
+            d({1: 'foo'})
+
+        d({1: 2})  # doesn't raise
+
+    def test_dict_type_with_recursive_item_types(self):
+        d = Dict(Dict({Unicode: List(Int)}), 'name')
+        with self.assertRaises(ValueError):
+            d({'foo': 'bar'})
+
+        with self.assertRaises(ValueError):
+            d({'foo': {'bar': 'baz'}})
+
+        with self.assertRaises(ValueError):
+            d({'foo': {'bar': ['baz']}})
+
+        d({'foo': {'bar': [1]}})  # doesn't raise
+
+    def test_list_type_basic(self):
+        l = List('name')
+        with self.assertRaises(ValueError):
+            l(['foo'])
+
+        l = List(Any, 'name')
+        l(['foo', 1])  # doesn't raise
+
+    def test_list_type_with_recursive_item_types(self):
+        l = List(Dict(List(Tuple((Unicode, Int)))), 'name')
+        with self.assertRaises(ValueError):
+            l(['foo'])
+
+        with self.assertRaises(ValueError):
+            l([{'foo': 'bar'}])
+
+        with self.assertRaises(ValueError):
+            l([{'foo': ['bar']}])
+
+        l([{'foo': [('bar', 1)]}])  # doesn't raise
+
+    def test_tuple_type_basic(self):
+        t = Tuple('name')
+        with self.assertRaises(ValueError):
+            t((1,))
+
+        t = Tuple(Any, 'name')
+        t((1,))  # doesn't raise
+
+    def test_tuple_type_with_tuple_item_type(self):
+        t = Tuple((Unicode, Int))
+        with self.assertRaises(ValueError):
+            t(('foo', 'bar'))
+
+        t(('foo', 1))  # doesn't raise
+
+    def test_tuple_type_with_recursive_item_types(self):
+        t = Tuple((Dict(List(Any)), List(Dict(Any)), Unicode), 'name')
+        with self.assertRaises(ValueError):
+            t(({'foo': 'bar'}, [{'foo': 'bar'}], 'foo'))
+
+        with self.assertRaises(ValueError):
+            t(({'foo': ['bar']}, ['foo'], 'foo'))
+
+        t(({'foo': ['bar']}, [{'foo': 'bar'}], 'foo'))  # doesn't raise
+
+
+if __name__ == '__main__':
+    mozunit.main()