Bug 1384570 - Part 1 - Merge localization content from multiple channels. r?Pike draft
authorStaś Małolepszy <stas@mozilla.com>
Tue, 05 Sep 2017 15:27:08 +0200
changeset 11699 bceee34aacb6ca188cf2ea661dc5285e0c620858
parent 11606 1dbfe3241a3b2e48f9cf76fc56681301c6db1ed6
child 11700 1be153f559045caa70eb1e5d4b856cb9f0624683
push id1794
push usersmalolepszy@mozilla.com
push dateFri, 22 Sep 2017 17:41:45 +0000
reviewersPike
bugs1384570
Bug 1384570 - Part 1 - Merge localization content from multiple channels. r?Pike MozReview-Commit-ID: EJZAGtY8R9f
cross-channel-l10n/mozxchannel/merge.py
cross-channel-l10n/test-requirements.txt
cross-channel-l10n/tests/test-merge-comments.py
cross-channel-l10n/tests/test-merge-dtd.py
cross-channel-l10n/tests/test-merge-ftl.py
cross-channel-l10n/tests/test-merge-messages.py
cross-channel-l10n/tests/test-merge-properties.py
cross-channel-l10n/tests/test-merge-whitespace.py
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/mozxchannel/merge.py
@@ -0,0 +1,53 @@
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from collections import OrderedDict
+
+from compare_locales import parser as cl
+from compare_locales.compare import AddRemove
+
+
+def merge_channels(name, *resources):
+    parser = cl.getParser(name)
+
+    if isinstance(parser, cl.FluentParser):
+        raise Exception("Fluent files (.ftl) are not supported (bug 1399055).")
+
+    def parse_resource(resource):
+        parser.readContents(resource)
+        pairs = [(get_key(entity), entity) for entity in parser.walk()]
+        return OrderedDict(pairs)
+
+    def get_key(entity):
+        if isinstance(entity, cl.Comment):
+            return entity.all
+        return entity.key
+
+    entities = reduce(merge_two, map(parse_resource, resources))
+
+    return serialize_legacy_resource(entities)
+
+
+def merge_two(newer, older):
+    diff = AddRemove()
+    diff.set_left(newer.keys())
+    diff.set_right(older.keys())
+
+    def get_entity(key):
+        return newer.get(key, older.get(key))
+
+    contents = [(key, get_entity(key)) for _, key in diff]
+    return OrderedDict(contents)
+
+
+def serialize_legacy_resource(entities):
+    return reduce(serialize_legacy_entity, entities.values(), "")
+
+
+def serialize_legacy_entity(acc, entity):
+    # Ensure there's a newline at the end of the resource content so far before
+    # we append the current entity's content.
+    if acc and not acc.endswith("\n"):
+        acc += "\n"
+
+    return acc + entity.all
--- a/cross-channel-l10n/test-requirements.txt
+++ b/cross-channel-l10n/test-requirements.txt
@@ -1,6 +1,11 @@
 -r prod-requirements.txt
 
 coverage==4.3.4 \
     --hash=sha256:ca36d83cd591d027952e5019149c4386e7058cd674bf8cb52dc622f768d689e9 \
     --hash=sha256:36407249a0b6669c6ad4425b0f29685579df745480c03afa70f101f09f4eead3 \
     --hash=sha256:f99066d76274800145a2e658026b30962eb5079346249197e88b55c9a7855e6a
+
+nose==1.3.7 \
+    --hash=sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a \
+    --hash=sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac \
+    --hash=sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-comments.py
@@ -0,0 +1,59 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeComments(unittest.TestCase):
+    name = "foo.properties"
+
+    def test_comment_added_in_first(self):
+        channels = ("""
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""")
+
+    def test_comment_still_in_last(self):
+        channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+# Bar Comment 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 2
+bar = Bar 1
+""")
+
+    def test_comment_changed(self):
+        channels = ("""
+foo = Foo 1
+# Bar Comment 1
+bar = Bar 1
+""", """
+foo = Foo 2
+# Bar Comment 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+# Bar Comment 2
+# Bar Comment 1
+bar = Bar 1
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-dtd.py
@@ -0,0 +1,20 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeDTD(unittest.TestCase):
+    name = "foo.dtd"
+
+    def test_no_changes(self):
+        channels = ("""
+<!ENTITY foo "Foo 1">
+""", """
+<!ENTITY foo "Foo 2">
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+<!ENTITY foo "Foo 1">
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-ftl.py
@@ -0,0 +1,21 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeFluent(unittest.TestCase):
+    name = "foo.ftl"
+
+    def test_no_support_for_now(self):
+        channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+        pattern = "Fluent files \(.ftl\) are not supported \(bug 1399055\)."
+        with self.assertRaisesRegexp(Exception, pattern):
+            merge_channels(self.name, *channels)
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-messages.py
@@ -0,0 +1,92 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeTwo(unittest.TestCase):
+    name = "foo.properties"
+
+    def test_no_changes(self):
+        channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
+
+    def test_message_added_in_first(self):
+        channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 1
+""")
+
+    def test_message_still_in_last(self):
+        channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 2
+""")
+
+    def test_message_reordered(self):
+        channels = ("""
+foo = Foo 1
+bar = Bar 1
+""", """
+bar = Bar 2
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 1
+""")
+
+
+class TestMergeThree(unittest.TestCase):
+    name = "foo.properties"
+
+    def test_no_changes(self):
+        channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""", """
+foo = Foo 3
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
+
+    def test_message_still_in_last(self):
+        channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""", """
+foo = Foo 3
+bar = Bar 3
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 3
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-properties.py
@@ -0,0 +1,20 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeProperties(unittest.TestCase):
+    name = "foo.properties"
+
+    def test_no_changes(self):
+        channels = ("""
+foo = Foo 1
+""", """
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+""")
new file mode 100644
--- /dev/null
+++ b/cross-channel-l10n/tests/test-merge-whitespace.py
@@ -0,0 +1,75 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+from mozxchannel.merge import merge_channels
+
+
+class TestMergeWhitespace(unittest.TestCase):
+    name = "foo.properties"
+
+    def test_trailing_spaces(self):
+        channels = ("""
+foo = Foo 1
+      """, """
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+      """)
+
+    def test_blank_lines_between_messages(self):
+        channels = ("""
+foo = Foo 1
+
+bar = Bar 1
+""", """
+foo = Foo 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+
+bar = Bar 1
+""")
+
+    def test_no_eol(self):
+        channels = ("""
+foo = Foo 1""", """
+foo = Foo 2
+bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+foo = Foo 1
+bar = Bar 2
+""")
+
+    def test_still_in_last_with_blank(self):
+        channels = ("""
+
+foo = Foo 1
+
+baz = Baz 1
+
+""", """
+
+foo = Foo 2
+
+bar = Bar 2
+
+baz = Baz 2
+
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), """
+
+foo = Foo 1
+
+bar = Bar 2
+
+baz = Baz 1
+
+""")