bug 1310980, part 5: fix DefinesParser, also add tests draft
authorAxel Hecht <axel@pike.org>
Fri, 21 Oct 2016 17:55:09 +0200
changeset 146 ce58f9cd9893c0f60643deacc6b19461433cc51f
parent 145 1429ed19667ba4bb7447db0ec28b46468567c087
child 147 72ff392e5c8d2bf696869b40eb26d4186b34baff
push id30
push useraxel@mozilla.com
push dateMon, 24 Oct 2016 14:52:08 +0000
bugs1310980
bug 1310980, part 5: fix DefinesParser, also add tests So yes, this parser didn't have tests yet at all. MozReview-Commit-ID: 8iEuHq1c3f0
compare_locales/parser.py
compare_locales/tests/test_defines.py
--- a/compare_locales/parser.py
+++ b/compare_locales/parser.py
@@ -447,28 +447,78 @@ class PropertiesParser(Parser):
                 return unichr(int(found['uni'][1:], 16))
             if found['nl']:
                 return ''
             return self.known_escapes.get(found['single'], found['single'])
         val = self.escape.sub(unescape, val)
         return val
 
 
+class DefinesInstruction(Entity):
+    '''Entity-like object representing processing instructions in inc files
+    '''
+    def __init__(self, ctx, span, pre_ws_span, def_span, val_span, post_span):
+        self.ctx = ctx
+        self.span = span
+        self.pre_ws_span = pre_ws_span
+        self.def_span = def_span
+        self.key_span = self.val_span = val_span
+        self.post_span = post_span
+        self.pp = lambda v: v
+
+    def __repr__(self):
+        return self.raw_val
+
+
 class DefinesParser(Parser):
     # can't merge, #unfilter needs to be the last item, which we don't support
     canMerge = False
 
     def __init__(self):
-        self.reKey = re.compile('^(\s*)'
-                                '(#define[ \t]+(\w+)[ \t]+(.*?))([ \t]*$\n?)',
+        self.reComment = re.compile(
+            '((?:[ \t]*\n)*)'
+            '((?:^# .*?(?:\n|\Z))+)'
+            '((?:[ \t]*(?:\n|\Z))*)', re.M)
+        self.reKey = re.compile('((?:[ \t]*\n)*)'
+                                '(#define[ \t]+(\w+)[ \t]+(.*?)(?:\n|\Z))'
+                                '((?:[ \t]*(?:\n|\Z))*)',
                                 re.M)
+        self.rePI = re.compile('((?:[ \t]*\n)*)'
+                               '(#(\w+)[ \t]+(.*?)(?:\n|\Z))'
+                               '((?:[ \t]*(?:\n|\Z))*)',
+                               re.M)
         self.reHeader = re.compile('^\s*(#(?!define\s).*\s*)*')
         self.reFooter = re.compile('\s*(#(?!define\s).*\s*)*$', re.M)
         Parser.__init__(self)
 
+    def getEntity(self, ctx, offset):
+        contents = ctx.contents
+        m = self.reComment.match(contents, offset)
+        if m:
+            offset = m.end()
+            self.last_comment = Comment(ctx, *[m.span(i) for i in xrange(4)])
+            return (self.last_comment, offset)
+        m = self.reKey.match(contents, offset)
+        if m:
+            offset = m.end()
+            return (self.createEntity(ctx, m), offset)
+        m = self.rePI.match(contents, offset)
+        if m:
+            offset = m.end()
+            return (DefinesInstruction(ctx, *[m.span(i) for i in xrange(5)]),
+                    offset)
+        junkend = None
+        for exp in (self.reComment, self.reKey, self.rePI):
+            m = exp.search(contents, offset)
+            if m:
+                junkend = min(junkend, m.start(1)) if junkend else m.start()
+        if junkend is not None:
+            return (Junk(ctx, (offset, junkend)), junkend)
+        return (None, offset)
+
 
 class IniSection(Entity):
     '''Entity-like object representing sections in ini files
     '''
     def __init__(self, ctx, span, pre_ws_span, def_span, val_span, post_span):
         self.ctx = ctx
         self.span = span
         self.pre_ws_span = pre_ws_span
copy from compare_locales/tests/test_ini.py
copy to compare_locales/tests/test_defines.py
--- a/compare_locales/tests/test_ini.py
+++ b/compare_locales/tests/test_defines.py
@@ -4,130 +4,67 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 
 from compare_locales.tests import ParserTestMixin
 
 
 mpl2 = '''\
-; This Source Code Form is subject to the terms of the Mozilla Public
-; License, v. 2.0. If a copy of the MPL was not distributed with this file,
-; You can obtain one at http://mozilla.org/MPL/2.0/.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
 '''
 
 
-class TestIniParser(ParserTestMixin, unittest.TestCase):
-
-    filename = 'foo.ini'
-
-    def testSimpleHeader(self):
-        self._test('''; This file is in the UTF-8 encoding
-[Strings]
-TitleText=Some Title
-''', (
-            ('Comment', 'UTF-8 encoding'),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),))
+class TestDefinesParser(ParserTestMixin, unittest.TestCase):
 
-    def testMPL2_Space_UTF(self):
-        self._test(mpl2 + '''
-; This file is in the UTF-8 encoding
-[Strings]
-TitleText=Some Title
-''', (
-            ('Comment', mpl2),
-            ('Comment', 'UTF-8'),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),))
+    filename = 'defines.inc'
 
-    def testMPL2_Space(self):
-        self._test(mpl2 + '''
-[Strings]
-TitleText=Some Title
-''', (
-            ('Comment', mpl2),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),))
-
-    def testMPL2_MultiSpace(self):
-        self._test(mpl2 + '''\
-
-; more comments
+    def testBrowser(self):
+        self._test(mpl2 + '''#filter emptyLines
 
-[Strings]
-TitleText=Some Title
-''', (
-            ('Comment', mpl2),
-            ('Comment', 'more comments'),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),))
-
-    def testMPL2_JunkBeforeCategory(self):
-        self._test(mpl2 + '''\
-Junk
-[Strings]
-TitleText=Some Title
-''', (
-            ('Comment', mpl2),
-            ('Junk', 'Junk'),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title')))
+#define MOZ_LANGPACK_CREATOR mozilla.org
 
-    def test_TrailingComment(self):
-        self._test(mpl2 + '''
-[Strings]
-TitleText=Some Title
-;Stray trailing comment
-''', (
-            ('Comment', mpl2),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),
-            ('Comment', 'Stray trailing')))
+# If non-English locales wish to credit multiple contributors, uncomment this
+# variable definition and use the format specified.
+# #define MOZ_LANGPACK_CONTRIBUTORS <em:contributor>Joe Solon</em:contributor>
 
-    def test_SpacedTrailingComments(self):
-        self._test(mpl2 + '''
-[Strings]
-TitleText=Some Title
-
-;Stray trailing comment
-;Second stray comment
+#unfilter emptyLines
 
 ''', (
             ('Comment', mpl2),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),
-            ('Comment', 'Second stray comment')))
+            ('DefinesInstruction', 'filter emptyLines'),
+            ('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
+            ('Comment', '#define'),
+            ('DefinesInstruction', 'unfilter emptyLines')))
+
+    def testBrowserWithContributors(self):
+        self._test(mpl2 + '''#filter emptyLines
 
-    def test_TrailingCommentsAndJunk(self):
-        self._test(mpl2 + '''
-[Strings]
-TitleText=Some Title
+#define MOZ_LANGPACK_CREATOR mozilla.org
 
-;Stray trailing comment
-Junk
-;Second stray comment
+# If non-English locales wish to credit multiple contributors, uncomment this
+# variable definition and use the format specified.
+#define MOZ_LANGPACK_CONTRIBUTORS <em:contributor>Joe Solon</em:contributor>
+
+#unfilter emptyLines
 
 ''', (
             ('Comment', mpl2),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),
-            ('Comment', 'Stray trailing'),
-            ('Junk', 'Junk'),
-            ('Comment', 'Second stray comment')))
+            ('DefinesInstruction', 'filter emptyLines'),
+            ('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
+            ('Comment', 'non-English'),
+            ('MOZ_LANGPACK_CONTRIBUTORS',
+             '<em:contributor>Joe Solon</em:contributor>'),
+            ('DefinesInstruction', 'unfilter emptyLines')))
 
-    def test_JunkInbetweenEntries(self):
-        self._test(mpl2 + '''
-[Strings]
-TitleText=Some Title
-
-Junk
+    def testToolkit(self):
+        self._test('''#define MOZ_LANG_TITLE English (US)
+''', (
+            ('MOZ_LANG_TITLE', 'English (US)'),))
 
-Good=other string
-''', (
-            ('Comment', mpl2),
-            ('IniSection', 'Strings'),
-            ('TitleText', 'Some Title'),
-            ('Junk', 'Junk'),
-            ('Good', 'other string')))
+    def testToolkitEmpty(self):
+        self._test('', tuple())
+
 
 if __name__ == '__main__':
     unittest.main()