bug 1199670, add l20n support to compare-locales draft
authorAxel Hecht <axel@pike.org>
Fri, 18 Nov 2016 16:04:30 +0100
changeset 248 71daf2befbf041c70f9d71feb50026c83d95e012
parent 245 8b6af885adb94191f241f2c75a0edb8927bc20bb
push id56
push useraxel@mozilla.com
push dateMon, 29 May 2017 14:54:00 +0000
bugs1199670
bug 1199670, add l20n support to compare-locales Uses https://github.com/l20n/python-l20n to do the actual parsing. MozReview-Commit-ID: 6hrfi1xsNNN
compare_locales/checks.py
compare_locales/parser.py
compare_locales/tests/test_ftl.py
compare_locales/tests/test_merge.py
--- a/compare_locales/checks.py
+++ b/compare_locales/checks.py
@@ -417,14 +417,52 @@ class DTDChecker(Checker):
                           u"or \\u0022, or put string in apostrophes."
                 else:
                     msg = u"Apostrophes in Android DTDs need escaping with "\
                           u"\\' or \\u0027, or use \u2019, or put string in "\
                           u"quotes."
                 yield ('error', m.end(0)+offset, msg, 'android')
 
 
+class L20nChecker(Checker):
+    '''Tests to run on l20n files.
+    '''
+    pattern = re.compile('.*\.ftl')
+
+    def check(self, refEnt, l10nEnt):
+        ref_entry = refEnt.entry
+        l10n_entry = l10nEnt.entry
+        # verify that values match, either both have a value or none
+        if ref_entry.value and not l10n_entry.value:
+            yield ('warning', l10n_entry._pos['start'],
+                   'Missing value', 'l20n')
+        if not ref_entry.value and l10n_entry.value:
+            yield ('warning', l10n_entry.value._pos['start'],
+                   'Obsolete value', 'l20n')
+        # verify that we're having the same set of attributes
+        # just the names matter
+        ref_attrs = set((attr.id.name for attr in ref_entry.attrs))
+        ref_pos = dict((attr.id.name, i)
+                       for i, attr in enumerate(ref_entry.attrs))
+        l10n_attrs = set((attr.id.name for attr in l10n_entry.attrs))
+        l10n_pos = dict((attr.id.name, i)
+                        for i, attr in enumerate(l10n_entry.attrs))
+        missing_attrs = sorted(ref_attrs - l10n_attrs,
+                               key=lambda k: ref_pos[k])
+        for name in missing_attrs:
+            yield ('warning', l10n_entry._pos['start'],
+                   'Missing attribute: ' + name, 'l20n')
+        obsolete_attrs = sorted(l10n_attrs - ref_attrs,
+                                key=lambda k: l10n_pos[k])
+        for name in obsolete_attrs:
+            attr = [_a for _a in l10n_entry.attrs if _a.id.name == name][0]
+            yield ('warning', attr._pos['start'],
+                   'Obsolete attribute: ' + name, 'l20n')
+
+
 def getChecker(file, reference=None, extra_tests=None):
     if PropertiesChecker.use(file):
         return PropertiesChecker(extra_tests)
     if DTDChecker.use(file):
         return DTDChecker(extra_tests, reference)
+    if L20nChecker.use(file):
+        return L20nChecker()
     return None
--- a/compare_locales/parser.py
+++ b/compare_locales/parser.py
@@ -2,16 +2,19 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import bisect
 import codecs
 import logging
 
+from l20n.format.parser import ParseContext as FTLParseContext
+from l20n.format import ast as l20n_ast
+
 __constructors = []
 
 
 class EntityBase(object):
     '''
     Abstraction layer for a localizable entity.
     Currently supported are grammars of the form:
 
@@ -561,12 +564,60 @@ class IniParser(Parser):
         m = self.reKey.match(contents, offset)
         if m:
             offset = m.end()
             return (self.createEntity(ctx, m), offset)
         return self.getTrailing(ctx, offset,
                                 self.reComment, self.reSection, self.reKey)
 
 
+class L20nParser(Parser):
+
+    class L20nEntity(Entity):
+        def pp(self, value):
+            return value
+        def __init__(self, ctx, entry):
+            self.ctx = ctx
+            self.key_span = (entry.id._pos['start'], entry.id._pos['end'])
+            val_end = (entry.traits[-1]._pos['end']
+                if entry.traits
+                else entry.value._pos['end'])
+            self.val_span = (entry.value._pos['start'], val_end)
+            self.entry = entry
+            self.span = (entry._pos['start'], entry._pos['end'])
+
+        @property
+        def all(self):
+            return self.contents[self.span[0]:self.span[1]]
+
+    def walk(self, onlyEntities=False):
+        if not self.ctx:
+            # loading file failed, or we just didn't load anything
+            return
+        ctx = self.ctx
+        contents = ctx.contents
+        _pos = l20n_ast.Node._pos
+        l20n_ast.Node._pos = True
+        parseContext = FTLParseContext(contents)
+        try:
+            [resource, errors] = parseContext.getResource()
+        finally:
+            l20n_ast.Node._pos = _pos
+        for entry in resource.entities():
+            if isinstance(entry, l20n_ast.Entity):
+                yield self.L20nEntity(self.ctx, entry)
+            elif isinstance(entry, l20n_ast.JunkEntry):
+                start = entry._pos['start']
+                # strip leading whitespace
+                start += re.match('\s*', entry.content).end()
+                end = entry._pos['end']
+                # strip trailing whitespace
+                ws, we = re.search('\s*$', entry.content).span()
+                end -= we - ws
+                yield Junk(self.ctx, (start, end))
+
+
 __constructors = [('\\.dtd$', DTDParser()),
                   ('\\.properties$', PropertiesParser()),
                   ('\\.ini$', IniParser()),
-                  ('\\.inc$', DefinesParser())]
+                  ('\\.inc$', DefinesParser()),
+                  ('\\.ftl$', L20nParser()),
+ ]
new file mode 100644
--- /dev/null
+++ b/compare_locales/tests/test_ftl.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from compare_locales.tests import ParserTestMixin
+
+
+mpl2 = '''\
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this file,
+; You can obtain one at http://mozilla.org/MPL/2.0/.
+'''
+
+
+class TestL20nParser(ParserTestMixin, unittest.TestCase):
+
+    filename = 'foo.ftl'
+
+    def test_simple_message(self):
+        self._test('foo = string value', (('foo', 'string value'),))
+
+    def test_section(self):
+        self._test('''
+[[ my_section ]]
+key = value
+''', (
+            ('key', 'value'),
+    ))
--- a/compare_locales/tests/test_merge.py
+++ b/compare_locales/tests/test_merge.py
@@ -282,10 +282,182 @@ class TestDTD(unittest.TestCase, Content
         mergefile = mozpath.join(self.tmp, "merge", "l10n.dtd")
         self.assertTrue(os.path.isfile(mergefile))
         p = getParser(mergefile)
         p.readFile(mergefile)
         [m, n] = p.parse()
         self.assertEqual(map(lambda e: e.key,  m), ["foo", "eff", "bar"])
 
 
+
+class TestL20n(unittest.TestCase):
+    maxDiff = None  # we got big dictionaries to compare
+
+    def reference(self, content):
+        self.ref = os.path.join(self.tmp, "en-reference.ftl")
+        open(self.ref, "w").write(content)
+
+    def localized(self, content):
+        self.l10n = os.path.join(self.tmp, "l10n.ftl")
+        open(self.l10n, "w").write(content)
+
+    def setUp(self):
+        self.tmp = mkdtemp()
+        os.mkdir(os.path.join(self.tmp, "merge"))
+        self.ref = self.l10n = None
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp)
+        del self.tmp
+        del self.ref
+        del self.l10n
+
+    def testGood(self):
+        self.reference("""\
+foo = fooVal
+bar = barVal
+eff = effVal
+""")
+        self.localized("""\
+foo = lFoo
+bar = lBar
+eff = lEff
+""")
+        cc = ContentComparer()
+        cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+        cc.compare(File(self.ref, "en-reference.ftl", ""),
+                   File(self.l10n, "l10n.ftl", ""))
+        self.assertDictEqual(
+            cc.observer.toJSON(),
+            {'summary':
+                {None: {
+                    'changed': 3
+                }},
+             'details': {}
+             }
+        )
+        # validate merge results, file shouldn't exist
+        self.assert_(
+            not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
+    def testMissing(self):
+        self.reference("""\
+foo = fooVal
+bar = barVal
+eff = effVal
+""")
+        self.localized("""foo = lFoo
+eff = lEff
+""")
+        cc = ContentComparer()
+        cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+        cc.compare(File(self.ref, "en-reference.ftl", ""),
+                   File(self.l10n, "l10n.ftl", ""))
+        self.assertDictEqual(
+            cc.observer.toJSON(),
+            {'details': {
+                'children': [
+                    ('l10n.ftl',
+                        {'value': {'missingEntity': [u'bar']}}
+                     )
+                ]},
+             'summary': {
+                None: {'changed': 2, 'missing': 1}
+            }}
+        )
+        # validate merge results
+        mergepath = os.path.join(cc.merge_stage, 'l10n.ftl')
+        p = getParser(mergepath)
+        p.readFile(mergepath)
+        entities, merged_map = p.parse()
+        self.assertEqual(sorted(merged_map.keys()), ['bar', 'eff', 'foo'])
+        # bar is missing, should be ref, foo and eff should be l10n
+        # check AST jsons to be sure
+        bar = entities[merged_map['bar']].entry.toJSON()
+        eff = entities[merged_map['eff']].entry.toJSON()
+        foo = entities[merged_map['foo']].entry.toJSON()
+        # parse ref
+        p.readFile(self.ref)
+        entities, _map = p.parse()
+        self.assertDictEqual(bar, entities[_map['bar']].entry.toJSON())
+        # parse l10n
+        p.readFile(self.l10n)
+        entities, _map = p.parse()
+        self.assertDictEqual(eff, entities[_map['eff']].entry.toJSON())
+        self.assertDictEqual(foo, entities[_map['foo']].entry.toJSON())
+
+    def testMismatchingAttributes(self):
+        self.reference("""
+foo = Foo
+bar = Bar
+  [tender] Attribute value
+eff = Eff
+""")
+        self.localized("""\
+foo = lFoo
+  [obsolete] attr
+bar = lBar
+eff = lEff
+""")
+        cc = ContentComparer()
+        cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+        cc.compare(File(self.ref, "en-reference.ftl", ""),
+                   File(self.l10n, "l10n.ftl", ""))
+        self.assertDictEqual(
+            cc.observer.toJSON(),
+            {'details': {
+                'children': [
+                    ('l10n.ftl',
+                        {'value': {
+                            'warning': [u'Missing attribute: tender at line 3,'
+                                        ' column 0 for bar',
+                                        u'Obsolete attribute: '
+                                        'obsolete at line 2, column 2 for foo']
+                        }}
+                     )
+                ]},
+             'summary': {
+                None: {'changed': 3, 'warnings': 2}
+            }}
+        )
+        # validate merge results, file shouldn't exist
+        self.assert_(
+            not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
+    def testMismatchingValues(self):
+        self.reference("""
+foo = Foo
+  [foottr] something
+bar
+  [tender] = Attribute value
+""")
+        self.localized("""\
+foo
+  [foottr] attr
+bar lBar
+  [tender] localized
+""")
+        cc = ContentComparer()
+        cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+        cc.compare(File(self.ref, "en-reference.ftl", ""),
+                   File(self.l10n, "l10n.ftl", ""))
+        self.assertDictEqual(
+            cc.observer.toJSON(),
+            {'details': {
+                'children': [
+                    ('l10n.ftl',
+                        {'value': {
+                            'warning': [
+                                u'Obsolete value at line 2, column 6 for bar',
+                                u'Missing value at line 1, column 0 for foo']
+                        }}
+                     )
+                ]},
+             'summary': {
+                None: {'changed': 2, 'warnings': 2}
+            }}
+        )
+        # validate merge results, file shouldn't exist
+        self.assert_(
+            not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
 if __name__ == '__main__':
     unittest.main()