Bug 1286075: improve dict merging support; r=gps draft
authorDustin J. Mitchell <dustin@mozilla.com>
Fri, 19 Aug 2016 18:12:40 +0000
changeset 412724 7ee2444782b1354d5a4edcfc55c3044b292fcc46
parent 412723 1d6360413bc5494fae792a3bca28f07a5bd42289
child 412725 58f1debe5219fc52ead2971718da932b63bb06d7
push id29252
push userdmitchell@mozilla.com
push dateMon, 12 Sep 2016 19:16:39 +0000
reviewersgps
bugs1286075
milestone51.0a1
Bug 1286075: improve dict merging support; r=gps MozReview-Commit-ID: D3691sf2LqZ
taskcluster/taskgraph/test/test_util_templates.py
taskcluster/taskgraph/util/templates.py
--- a/taskcluster/taskgraph/test/test_util_templates.py
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -3,16 +3,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 import mozunit
 import textwrap
 from taskgraph.util.templates import (
+    merge_to,
+    merge,
     Templates,
     TemplatesException
 )
 
 files = {}
 files['/fixtures/circular.yml'] = textwrap.dedent("""\
     $inherits:
       from: 'circular_ref.yml'
@@ -177,10 +179,54 @@ class TemplatesTest(unittest.TestCase):
                     'list': ['baz', 'bar']
                 },
                 'level': 2,
             },
             'was_list': {'replaced': True}
         })
 
 
+class MergeTest(unittest.TestCase):
+
+    def test_merge_to_dicts(self):
+        source = {'a': 1, 'b': 2}
+        dest = {'b': '20', 'c': 30}
+        expected = {
+            'a': 1,   # source only
+            'b': 2,   # source overrides dest
+            'c': 30,  # dest only
+        }
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge_to_lists(self):
+        source = {'x': [3, 4]}
+        dest = {'x': [1, 2]}
+        expected = {'x': [1, 2, 3, 4]}  # dest first
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge_diff_types(self):
+        source = {'x': [1, 2]}
+        dest = {'x': 'abc'}
+        expected = {'x': [1, 2]}  # source wins
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge(self):
+        first = {'a': 1, 'b': 2, 'd': 11}
+        second = {'b': 20, 'c': 30}
+        third = {'c': 300, 'd': 400}
+        expected = {
+            'a': 1,
+            'b': 20,
+            'c': 300,
+            'd': 400,
+        }
+        self.assertEqual(merge(first, second, third), expected)
+
+        # inputs haven't changed..
+        self.assertEqual(first, {'a': 1, 'b': 2, 'd': 11})
+        self.assertEqual(second, {'b': 20, 'c': 30})
+        self.assertEqual(third, {'c': 300, 'd': 400})
+
 if __name__ == '__main__':
     mozunit.main()
--- a/taskcluster/taskgraph/util/templates.py
+++ b/taskcluster/taskgraph/util/templates.py
@@ -1,23 +1,27 @@
 import os
 
 import pystache
 import yaml
+import copy
 
 # Key used in template inheritance...
 INHERITS_KEY = '$inherits'
 
 
 def merge_to(source, dest):
     '''
     Merge dict and arrays (override scalar values)
 
+    Keys from source override keys from dest, and elements from lists in source
+    are appended to lists in dest.
+
     :param dict source: to copy from
-    :param dict dest: to copy to.
+    :param dict dest: to copy to (modified in place)
     '''
 
     for key, value in source.items():
         # Override mismatching or empty types
         if type(value) != type(dest.get(key)):  # noqa
             dest[key] = source[key]
             continue
 
@@ -30,16 +34,29 @@ def merge_to(source, dest):
             dest[key] = dest[key] + source[key]
             continue
 
         dest[key] = source[key]
 
     return dest
 
 
+def merge(*objects):
+    '''
+    Merge the given objects, using the semantics described for merge_to, with
+    objects later in the list taking precedence.  From an inheritance
+    perspective, "parents" should be listed before "children".
+
+    Returns the result without modifying any arguments.
+    '''
+    if len(objects) == 1:
+        return copy.deepcopy(objects[0])
+    return merge_to(objects[-1], merge(*objects[:-1]))
+
+
 class TemplatesException(Exception):
     pass
 
 
 class Templates():
     '''
     The taskcluster integration makes heavy use of yaml to describe tasks this
     class handles the loading/rendering.