Bug 1399055 - Support Fluent in cross-channel localization. r=Pike draft
authorStaś Małolepszy <stas@mozilla.com>
Thu, 05 Oct 2017 16:34:27 +0200
changeset 358 3f66c768bf6bd7b6c85677ddc096b55afb3d7a5b
parent 357 b85ebfa4932420aed41c119aa6693b82b698c857
push id121
push usersmalolepszy@mozilla.com
push dateMon, 09 Oct 2017 10:31:08 +0000
reviewersPike
bugs1399055
Bug 1399055 - Support Fluent in cross-channel localization. r=Pike MozReview-Commit-ID: 7ZgDBEbzdjG
compare_locales/merge.py
compare_locales/parser.py
compare_locales/tests/test_merge_ftl.py
--- a/compare_locales/merge.py
+++ b/compare_locales/merge.py
@@ -18,20 +18,16 @@ class MergeNotSupportedError(ValueError)
 
 def merge_channels(name, *resources):
     try:
         parser = cl.getParser(name)
     except UserWarning:
         raise MergeNotSupportedError(
             'Unsupported file format ({}).'.format(name))
 
-    if isinstance(parser, cl.FluentParser):
-        raise MergeNotSupportedError(
-            'Fluent files (.ftl) are not supported (bug 1399055).')
-
     # A map of comments to the keys of entities they belong to.
     comments = {}
 
     def parse_resource(resource):
         # The counter dict keeps track of number of identical comments.
         counter = defaultdict(int)
         parser.readContents(resource)
         pairs = [get_key_value(entity, counter) for entity in parser.walk()]
@@ -85,16 +81,17 @@ def merge_two(comments, newer, older):
         if isinstance(entity, cl.Comment) and entity in comments:
             next_entity = newer.get(comments[entity], None)
             if next_entity is not None and next_entity.pre_comment:
                 # We'll prune this before returning the merged result.
                 return None
 
         return entity
 
+    # Create a flat sequence of all entities in order reported by AddRemove.
     contents = [(key, get_entity(key)) for _, key in diff]
 
     def prune(acc, cur):
         _, entity = cur
         if entity is None:
             # Prune Nones which stand for duplicated comments.
             return acc
 
--- a/compare_locales/parser.py
+++ b/compare_locales/parser.py
@@ -583,16 +583,22 @@ class FluentEntity(Entity):
 
         if entry.value is not None:
             self.val_span = (entry.value.span.start, entry.value.span.end)
         else:
             self.val_span = (0, 0)
 
         self.entry = entry
 
+        # EntityBase instances are expected to have pre_comment. It's used by
+        # other formats to associate a Comment with an Entity. FluentEntities
+        # don't need it because message comments are part of the entry AST and
+        # are not separate Comment instances.
+        self.pre_comment = None
+
     _word_count = None
 
     def count_words(self):
         if self._word_count is None:
             self._word_count = 0
 
             def count_words(node):
                 if isinstance(node, ftl.TextElement):
--- a/compare_locales/tests/test_merge_ftl.py
+++ b/compare_locales/tests/test_merge_ftl.py
@@ -4,19 +4,313 @@
 
 import unittest
 from compare_locales.merge import merge_channels
 
 
 class TestMergeFluent(unittest.TestCase):
     name = "foo.ftl"
 
-    def test_no_support_for_now(self):
+    def test_no_changes(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+""")
+
+    def test_attribute_in_first(self):
+        channels = (b"""
+foo = Foo 1
+    .attr = Attr 1
+""", b"""
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+    .attr = Attr 1
+""")
+
+    def test_attribute_in_last(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+foo = Foo 2
+    .attr = Attr 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+""")
+
+    def test_attribute_changed(self):
+        channels = (b"""
+foo = Foo 1
+    .attr = Attr 1
+""", b"""
+foo = Foo 2
+    .attr = Attr 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+    .attr = Attr 1
+""")
+
+    def test_tag_in_first(self):
+        channels = (b"""
+foo = Foo 1
+    #tag
+""", b"""
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+    #tag
+""")
+
+    def test_tag_in_last(self):
         channels = (b"""
 foo = Foo 1
-bar = Bar 1
+""", b"""
+foo = Foo 2
+    #tag
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+""")
+
+    def test_tag_changed(self):
+        channels = (b"""
+foo = Foo 1
+    #tag1
+""", b"""
+foo = Foo 2
+    #tag2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+    #tag1
+""")
+
+    def test_section_in_first(self):
+        channels = (b"""
+[[ Section 1 ]]
+foo = Foo 1
+""", b"""
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+[[ Section 1 ]]
+foo = Foo 1
+""")
+
+    def test_section_in_last(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+[[ Section 2 ]]
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+[[ Section 2 ]]
+foo = Foo 1
+""")
+
+    def test_section_changed(self):
+        channels = (b"""
+[[ Section 1 ]]
+foo = Foo 1
+""", b"""
+[[ Section 2 ]]
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+[[ Section 2 ]]
+[[ Section 1 ]]
+foo = Foo 1
+""")
+
+    def test_message_comment_in_first(self):
+        channels = (b"""
+// Comment 1
+foo = Foo 1
 """, b"""
 foo = Foo 2
-bar = Bar 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Comment 1
+foo = Foo 1
+""")
+
+    def test_message_comment_in_last(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+// Comment 2
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+""")
+
+    def test_message_comment_changed(self):
+        channels = (b"""
+// Comment 1
+foo = Foo 1
+""", b"""
+// Comment 2
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Comment 1
+foo = Foo 1
+""")
+
+    def test_section_comment_in_first(self):
+        channels = (b"""
+// Comment 1
+[[ Section ]]
+""", b"""
+[[ Section ]]
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Comment 1
+[[ Section ]]
+""")
+
+    def test_section_comment_in_last(self):
+        channels = (b"""
+[[ Section ]]
+""", b"""
+// Comment 2
+[[ Section ]]
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+[[ Section ]]
+""")
+
+    def test_section_comment_changed(self):
+        channels = (b"""
+// Comment 1
+[[ Section ]]
+""", b"""
+// Comment 2
+[[ Section ]]
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Comment 1
+[[ Section ]]
+""")
+
+    def test_standalone_comment_in_first(self):
+        channels = (b"""
+foo = Foo 1
+
+// Comment 1
+""", b"""
+foo = Foo 2
 """)
-        pattern = "Fluent files \(.ftl\) are not supported \(bug 1399055\)."
-        with self.assertRaisesRegexp(Exception, pattern):
-            merge_channels(self.name, *channels)
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+
+// Comment 1
+""")
+
+    def test_standalone_comment_in_last(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+foo = Foo 2
+
+// Comment 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+
+// Comment 2
+""")
+
+    def test_standalone_comment_changed(self):
+        channels = (b"""
+foo = Foo 1
+
+// Comment 1
+""", b"""
+foo = Foo 2
+
+// Comment 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+foo = Foo 1
+
+// Comment 2
+
+// Comment 1
+""")
+
+    def test_resource_comment_in_first(self):
+        channels = (b"""
+// Resource Comment 1
+
+foo = Foo 1
+""", b"""
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Resource Comment 1
+
+foo = Foo 1
+""")
+
+    def test_resource_comment_in_last(self):
+        channels = (b"""
+foo = Foo 1
+""", b"""
+// Resource Comment 1
+
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Resource Comment 1
+
+foo = Foo 1
+""")
+
+    def test_resource_comment_changed(self):
+        channels = (b"""
+// Resource Comment 1
+
+foo = Foo 1
+""", b"""
+// Resource Comment 2
+
+foo = Foo 2
+""")
+        self.assertEqual(
+            merge_channels(self.name, *channels), b"""
+// Resource Comment 2
+
+// Resource Comment 1
+
+foo = Foo 1
+""")