bug 1199670, add l20n support to compare-locales
Uses https://github.com/l20n/python-l20n to do the actual parsing.
MozReview-Commit-ID: 6hrfi1xsNNN
--- a/compare_locales/checks.py
+++ b/compare_locales/checks.py
@@ -417,14 +417,52 @@ class DTDChecker(Checker):
u"or \\u0022, or put string in apostrophes."
else:
msg = u"Apostrophes in Android DTDs need escaping with "\
u"\\' or \\u0027, or use \u2019, or put string in "\
u"quotes."
yield ('error', m.end(0)+offset, msg, 'android')
+class L20nChecker(Checker):
+ '''Tests to run on l20n files.
+ '''
+ pattern = re.compile('.*\.ftl')
+
+ def check(self, refEnt, l10nEnt):
+ ref_entry = refEnt.entry
+ l10n_entry = l10nEnt.entry
+ # verify that values match, either both have a value or none
+ if ref_entry.value and not l10n_entry.value:
+ yield ('warning', l10n_entry._pos['start'],
+ 'Missing value', 'l20n')
+ if not ref_entry.value and l10n_entry.value:
+ yield ('warning', l10n_entry.value._pos['start'],
+ 'Obsolete value', 'l20n')
+ # verify that we're having the same set of attributes
+ # just the names matter
+ ref_attrs = set((attr.id.name for attr in ref_entry.attrs))
+ ref_pos = dict((attr.id.name, i)
+ for i, attr in enumerate(ref_entry.attrs))
+ l10n_attrs = set((attr.id.name for attr in l10n_entry.attrs))
+ l10n_pos = dict((attr.id.name, i)
+ for i, attr in enumerate(l10n_entry.attrs))
+ missing_attrs = sorted(ref_attrs - l10n_attrs,
+ key=lambda k: ref_pos[k])
+ for name in missing_attrs:
+ yield ('warning', l10n_entry._pos['start'],
+ 'Missing attribute: ' + name, 'l20n')
+ obsolete_attrs = sorted(l10n_attrs - ref_attrs,
+ key=lambda k: l10n_pos[k])
+ for name in obsolete_attrs:
+ attr = [_a for _a in l10n_entry.attrs if _a.id.name == name][0]
+ yield ('warning', attr._pos['start'],
+ 'Obsolete attribute: ' + name, 'l20n')
+
+
def getChecker(file, reference=None, extra_tests=None):
if PropertiesChecker.use(file):
return PropertiesChecker(extra_tests)
if DTDChecker.use(file):
return DTDChecker(extra_tests, reference)
+ if L20nChecker.use(file):
+ return L20nChecker()
return None
--- a/compare_locales/parser.py
+++ b/compare_locales/parser.py
@@ -2,16 +2,19 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import re
import bisect
import codecs
import logging
+from l20n.format.parser import ParseContext as FTLParseContext
+from l20n.format import ast as l20n_ast
+
__constructors = []
class EntityBase(object):
'''
Abstraction layer for a localizable entity.
Currently supported are grammars of the form:
@@ -561,12 +564,60 @@ class IniParser(Parser):
m = self.reKey.match(contents, offset)
if m:
offset = m.end()
return (self.createEntity(ctx, m), offset)
return self.getTrailing(ctx, offset,
self.reComment, self.reSection, self.reKey)
+class L20nParser(Parser):
+
+ class L20nEntity(Entity):
+ def pp(self, value):
+ return value
+ def __init__(self, ctx, entry):
+ self.ctx = ctx
+ self.key_span = (entry.id._pos['start'], entry.id._pos['end'])
+ val_end = (entry.traits[-1]._pos['end']
+ if entry.traits
+ else entry.value._pos['end'])
+ self.val_span = (entry.value._pos['start'], val_end)
+ self.entry = entry
+ self.span = (entry._pos['start'], entry._pos['end'])
+
+ @property
+ def all(self):
+ return self.contents[self.span[0]:self.span[1]]
+
+ def walk(self, onlyEntities=False):
+ if not self.ctx:
+ # loading file failed, or we just didn't load anything
+ return
+ ctx = self.ctx
+ contents = ctx.contents
+ _pos = l20n_ast.Node._pos
+ l20n_ast.Node._pos = True
+ parseContext = FTLParseContext(contents)
+ try:
+ [resource, errors] = parseContext.getResource()
+ finally:
+ l20n_ast.Node._pos = _pos
+ for entry in resource.entities():
+ if isinstance(entry, l20n_ast.Entity):
+ yield self.L20nEntity(self.ctx, entry)
+ elif isinstance(entry, l20n_ast.JunkEntry):
+ start = entry._pos['start']
+ # strip leading whitespace
+ start += re.match('\s*', entry.content).end()
+ end = entry._pos['end']
+ # strip trailing whitespace
+ ws, we = re.search('\s*$', entry.content).span()
+ end -= we - ws
+ yield Junk(self.ctx, (start, end))
+
+
__constructors = [('\\.dtd$', DTDParser()),
('\\.properties$', PropertiesParser()),
('\\.ini$', IniParser()),
- ('\\.inc$', DefinesParser())]
+ ('\\.inc$', DefinesParser()),
+ ('\\.ftl$', L20nParser()),
+ ]
new file mode 100644
--- /dev/null
+++ b/compare_locales/tests/test_ftl.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from compare_locales.tests import ParserTestMixin
+
+
+mpl2 = '''\
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this file,
+; You can obtain one at http://mozilla.org/MPL/2.0/.
+'''
+
+
+class TestL20nParser(ParserTestMixin, unittest.TestCase):
+
+ filename = 'foo.ftl'
+
+ def test_simple_message(self):
+ self._test('foo = string value', (('foo', 'string value'),))
+
+ def test_section(self):
+ self._test('''
+[[ my_section ]]
+key = value
+''', (
+ ('key', 'value'),
+ ))
--- a/compare_locales/tests/test_merge.py
+++ b/compare_locales/tests/test_merge.py
@@ -282,10 +282,182 @@ class TestDTD(unittest.TestCase, Content
mergefile = mozpath.join(self.tmp, "merge", "l10n.dtd")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["foo", "eff", "bar"])
+
+class TestL20n(unittest.TestCase):
+ maxDiff = None # we got big dictionaries to compare
+
+ def reference(self, content):
+ self.ref = os.path.join(self.tmp, "en-reference.ftl")
+ open(self.ref, "w").write(content)
+
+ def localized(self, content):
+ self.l10n = os.path.join(self.tmp, "l10n.ftl")
+ open(self.l10n, "w").write(content)
+
+ def setUp(self):
+ self.tmp = mkdtemp()
+ os.mkdir(os.path.join(self.tmp, "merge"))
+ self.ref = self.l10n = None
+
+ def tearDown(self):
+ shutil.rmtree(self.tmp)
+ del self.tmp
+ del self.ref
+ del self.l10n
+
+ def testGood(self):
+ self.reference("""\
+foo = fooVal
+bar = barVal
+eff = effVal
+""")
+ self.localized("""\
+foo = lFoo
+bar = lBar
+eff = lEff
+""")
+ cc = ContentComparer()
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.ftl", ""),
+ File(self.l10n, "l10n.ftl", ""))
+ self.assertDictEqual(
+ cc.observer.toJSON(),
+ {'summary':
+ {None: {
+ 'changed': 3
+ }},
+ 'details': {}
+ }
+ )
+ # validate merge results, file shouldn't exist
+ self.assert_(
+ not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
+ def testMissing(self):
+ self.reference("""\
+foo = fooVal
+bar = barVal
+eff = effVal
+""")
+ self.localized("""foo = lFoo
+eff = lEff
+""")
+ cc = ContentComparer()
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.ftl", ""),
+ File(self.l10n, "l10n.ftl", ""))
+ self.assertDictEqual(
+ cc.observer.toJSON(),
+ {'details': {
+ 'children': [
+ ('l10n.ftl',
+ {'value': {'missingEntity': [u'bar']}}
+ )
+ ]},
+ 'summary': {
+ None: {'changed': 2, 'missing': 1}
+ }}
+ )
+ # validate merge results
+ mergepath = os.path.join(cc.merge_stage, 'l10n.ftl')
+ p = getParser(mergepath)
+ p.readFile(mergepath)
+ entities, merged_map = p.parse()
+ self.assertEqual(sorted(merged_map.keys()), ['bar', 'eff', 'foo'])
+ # bar is missing, should be ref, foo and eff should be l10n
+ # check AST jsons to be sure
+ bar = entities[merged_map['bar']].entry.toJSON()
+ eff = entities[merged_map['eff']].entry.toJSON()
+ foo = entities[merged_map['foo']].entry.toJSON()
+ # parse ref
+ p.readFile(self.ref)
+ entities, _map = p.parse()
+ self.assertDictEqual(bar, entities[_map['bar']].entry.toJSON())
+ # parse l10n
+ p.readFile(self.l10n)
+ entities, _map = p.parse()
+ self.assertDictEqual(eff, entities[_map['eff']].entry.toJSON())
+ self.assertDictEqual(foo, entities[_map['foo']].entry.toJSON())
+
+ def testMismatchingAttributes(self):
+ self.reference("""
+foo = Foo
+bar = Bar
+ [tender] Attribute value
+eff = Eff
+""")
+ self.localized("""\
+foo = lFoo
+ [obsolete] attr
+bar = lBar
+eff = lEff
+""")
+ cc = ContentComparer()
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.ftl", ""),
+ File(self.l10n, "l10n.ftl", ""))
+ self.assertDictEqual(
+ cc.observer.toJSON(),
+ {'details': {
+ 'children': [
+ ('l10n.ftl',
+ {'value': {
+ 'warning': [u'Missing attribute: tender at line 3,'
+ ' column 0 for bar',
+ u'Obsolete attribute: '
+ 'obsolete at line 2, column 2 for foo']
+ }}
+ )
+ ]},
+ 'summary': {
+ None: {'changed': 3, 'warnings': 2}
+ }}
+ )
+ # validate merge results, file shouldn't exist
+ self.assert_(
+ not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
+ def testMismatchingValues(self):
+ self.reference("""
+foo = Foo
+ [foottr] something
+bar
+ [tender] = Attribute value
+""")
+ self.localized("""\
+foo
+ [foottr] attr
+bar lBar
+ [tender] localized
+""")
+ cc = ContentComparer()
+ cc.set_merge_stage(os.path.join(self.tmp, "merge"))
+ cc.compare(File(self.ref, "en-reference.ftl", ""),
+ File(self.l10n, "l10n.ftl", ""))
+ self.assertDictEqual(
+ cc.observer.toJSON(),
+ {'details': {
+ 'children': [
+ ('l10n.ftl',
+ {'value': {
+ 'warning': [
+ u'Obsolete value at line 2, column 6 for bar',
+ u'Missing value at line 1, column 0 for foo']
+ }}
+ )
+ ]},
+ 'summary': {
+ None: {'changed': 2, 'warnings': 2}
+ }}
+ )
+ # validate merge results, file shouldn't exist
+ self.assert_(
+ not os.path.exists(os.path.join(cc.merge_stage, 'l10n.ftl')))
+
if __name__ == '__main__':
unittest.main()