Bug 1286075: improve dict merging support; r=gps
MozReview-Commit-ID: D3691sf2LqZ
--- a/taskcluster/taskgraph/test/test_util_templates.py
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -3,16 +3,18 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import unittest
import mozunit
import textwrap
from taskgraph.util.templates import (
+ merge_to,
+ merge,
Templates,
TemplatesException
)
files = {}
files['/fixtures/circular.yml'] = textwrap.dedent("""\
$inherits:
from: 'circular_ref.yml'
@@ -177,10 +179,54 @@ class TemplatesTest(unittest.TestCase):
'list': ['baz', 'bar']
},
'level': 2,
},
'was_list': {'replaced': True}
})
+class MergeTest(unittest.TestCase):
+
+ def test_merge_to_dicts(self):
+ source = {'a': 1, 'b': 2}
+ dest = {'b': '20', 'c': 30}
+ expected = {
+ 'a': 1, # source only
+ 'b': 2, # source overrides dest
+ 'c': 30, # dest only
+ }
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge_to_lists(self):
+ source = {'x': [3, 4]}
+ dest = {'x': [1, 2]}
+ expected = {'x': [1, 2, 3, 4]} # dest first
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge_diff_types(self):
+ source = {'x': [1, 2]}
+ dest = {'x': 'abc'}
+ expected = {'x': [1, 2]} # source wins
+ self.assertEqual(merge_to(source, dest), expected)
+ self.assertEqual(dest, expected)
+
+ def test_merge(self):
+ first = {'a': 1, 'b': 2, 'd': 11}
+ second = {'b': 20, 'c': 30}
+ third = {'c': 300, 'd': 400}
+ expected = {
+ 'a': 1,
+ 'b': 20,
+ 'c': 300,
+ 'd': 400,
+ }
+ self.assertEqual(merge(first, second, third), expected)
+
+ # inputs haven't changed..
+ self.assertEqual(first, {'a': 1, 'b': 2, 'd': 11})
+ self.assertEqual(second, {'b': 20, 'c': 30})
+ self.assertEqual(third, {'c': 300, 'd': 400})
+
if __name__ == '__main__':
mozunit.main()
--- a/taskcluster/taskgraph/util/templates.py
+++ b/taskcluster/taskgraph/util/templates.py
@@ -1,23 +1,27 @@
import os
import pystache
import yaml
+import copy
# Key used in template inheritance...
INHERITS_KEY = '$inherits'
def merge_to(source, dest):
'''
Merge dict and arrays (override scalar values)
+ Keys from source override keys from dest, and elements from lists in source
+ are appended to lists in dest.
+
:param dict source: to copy from
- :param dict dest: to copy to.
+ :param dict dest: to copy to (modified in place)
'''
for key, value in source.items():
# Override mismatching or empty types
if type(value) != type(dest.get(key)): # noqa
dest[key] = source[key]
continue
@@ -30,16 +34,29 @@ def merge_to(source, dest):
dest[key] = dest[key] + source[key]
continue
dest[key] = source[key]
return dest
+def merge(*objects):
+ '''
+ Merge the given objects, using the semantics described for merge_to, with
+ objects later in the list taking precedence. From an inheritance
+ perspective, "parents" should be listed before "children".
+
+ Returns the result without modifying any arguments.
+ '''
+ if len(objects) == 1:
+ return copy.deepcopy(objects[0])
+ return merge_to(objects[-1], merge(*objects[:-1]))
+
+
class TemplatesException(Exception):
pass
class Templates():
'''
The taskcluster integration makes heavy use of yaml to describe tasks this
class handles the loading/rendering.