Bug 1384570 - Part 1 - Merge localization content from multiple channels. r?Pike
MozReview-Commit-ID: EJZAGtY8R9f
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/mozxchannel/merge.py
@@ -0,0 +1,53 @@
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from collections import OrderedDict
+
+from compare_locales import parser as cl
+from compare_locales.compare import AddRemove
+
+
+def merge_channels(name, *resources):
+ parser = cl.getParser(name)
+
+ if isinstance(parser, cl.FluentParser):
+ raise Exception("Fluent files (.ftl) are not supported (bug 1399055).")
+
+ def parse_resource(resource):
+ parser.readContents(resource)
+ pairs = [(get_key(entity), entity) for entity in parser.walk()]
+ return OrderedDict(pairs)
+
+ def get_key(entity):
+ if isinstance(entity, cl.Comment):
+ return entity.all
+ return entity.key
+
+ entities = reduce(merge_two, map(parse_resource, resources))
+
+ return serialize_legacy_resource(entities)
+
+
+def merge_two(newer, older):
+ diff = AddRemove()
+ diff.set_left(newer.keys())
+ diff.set_right(older.keys())
+
+ def get_entity(key):
+ return newer.get(key, older.get(key))
+
+ contents = [(key, get_entity(key)) for _, key in diff]
+ return OrderedDict(contents)
+
+
+def serialize_legacy_resource(entities):
+ return reduce(serialize_legacy_entity, entities.values(), "")
+
+
+def serialize_legacy_entity(acc, entity):
+ # Ensure there's a newline at the end of the resource content so far before
+ # we append the current entity's content.
+ if acc and not acc.endswith("\n"):
+ acc += "\n"
+
+ return acc + entity.all
--- a/cross-channel-l10n/test-requirements.txt
+++ b/cross-channel-l10n/test-requirements.txt
@@ -1,6 +1,11 @@
-r prod-requirements.txt
coverage==4.3.4 \
--hash=sha256:ca36d83cd591d027952e5019149c4386e7058cd674bf8cb52dc622f768d689e9 \
--hash=sha256:36407249a0b6669c6ad4425b0f29685579df745480c03afa70f101f09f4eead3 \
--hash=sha256:f99066d76274800145a2e658026b30962eb5079346249197e88b55c9a7855e6a
+
+nose==1.3.7 \
+ --hash=sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a \
+ --hash=sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac \
+ --hash=sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-comments.py
@@ -0,0 +1,59 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeComments(unittest.TestCase):
+ name = "foo.properties"
+
+ def test_comment_added_in_first(self):
+ channels = ("""
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""")
+
+ def test_comment_still_in_last(self):
+ channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+# Bar Comment 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 2
+bar = Bar 1
+""")
+
+ def test_comment_changed(self):
+ channels = ("""
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""", """
+foo = Foo 2
+# Bar Comment 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 2
+# Bar Comment 1
+bar = Bar 1
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-dtd.py
@@ -0,0 +1,20 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeDTD(unittest.TestCase):
+ name = "foo.dtd"
+
+ def test_no_changes(self):
+ channels = ("""
+<!ENTITY foo "Foo 1">
+""", """
+<!ENTITY foo "Foo 2">
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+<!ENTITY foo "Foo 1">
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-ftl.py
@@ -0,0 +1,21 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeFluent(unittest.TestCase):
+ name = "foo.ftl"
+
+ def test_no_support_for_now(self):
+ channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+ pattern = "Fluent files \(.ftl\) are not supported \(bug 1399055\)."
+ with self.assertRaisesRegexp(Exception, pattern):
+ merge_channels(self.name, *channels)
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-messages.py
@@ -0,0 +1,92 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeTwo(unittest.TestCase):
+ name = "foo.properties"
+
+ def test_no_changes(self):
+ channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
+
+ def test_message_added_in_first(self):
+ channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 1
+""")
+
+ def test_message_still_in_last(self):
+ channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 2
+""")
+
+ def test_message_reordered(self):
+ channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+bar = Bar 2
+foo = Foo 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 1
+""")
+
+
+class TestMergeThree(unittest.TestCase):
+ name = "foo.properties"
+
+ def test_no_changes(self):
+ channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""", """
+foo = Foo 3
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
+
+ def test_message_still_in_last(self):
+ channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""", """
+foo = Foo 3
+bar = Bar 3
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 3
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-properties.py
@@ -0,0 +1,20 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeProperties(unittest.TestCase):
+ name = "foo.properties"
+
+ def test_no_changes(self):
+ channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-whitespace.py
@@ -0,0 +1,75 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeWhitespace(unittest.TestCase):
+ name = "foo.properties"
+
+ def test_trailing_spaces(self):
+ channels = ("""
+foo = Foo 1
+ """, """
+foo = Foo 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+ """)
+
+ def test_blank_lines_between_messages(self):
+ channels = ("""
+foo = Foo 1
+
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+
+bar = Bar 1
+""")
+
+ def test_no_eol(self):
+ channels = ("""
+foo = Foo 1""", """
+foo = Foo 2
+bar = Bar 2
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 2
+""")
+
+ def test_still_in_last_with_blank(self):
+ channels = ("""
+
+foo = Foo 1
+
+baz = Baz 1
+
+""", """
+
+foo = Foo 2
+
+bar = Bar 2
+
+baz = Baz 2
+
+""")
+ self.assertEqual(
+ merge_channels(self.name, *channels), """
+
+foo = Foo 1
+
+bar = Bar 2
+
+baz = Baz 1
+
+""")