Bug 1399057 - Yield Sections and Comments from FluentParser.walk. r=Pike
MozReview-Commit-ID: 1Joi7B5MZZN
--- a/compare_locales/parser.py
+++ b/compare_locales/parser.py
@@ -621,29 +621,53 @@ class FluentEntity(Entity):
return self.position(pos)
@property
def attributes(self):
for attr_node in self.entry.attributes:
yield FluentAttribute(self, attr_node)
+class FluentSection(EntityBase):
+ def __init__(self, ctx, entry):
+ self.entry = entry
+ self.ctx = ctx
+
+ self.span = (entry.span.start, entry.span.end)
+ self.key_span = self.val_span = (
+ entry.name.span.start, entry.name.span.end)
+
+
class FluentParser(Parser):
capabilities = CAN_SKIP
def __init__(self):
super(FluentParser, self).__init__()
self.ftl_parser = FTLParser()
def walk(self, only_localizable=False):
if not self.ctx:
# loading file failed, or we just didn't load anything
return
+
resource = self.ftl_parser.parse(self.ctx.contents)
- last_span_end = resource.comment.span.end if resource.comment else 0
+
+ if resource.comment:
+ last_span_end = resource.comment.span.end
+
+ if not only_localizable:
+ if 0 < resource.comment.span.start:
+ yield Whitespace(
+ self.ctx, (0, resource.comment.span.start))
+ yield Comment(
+ self.ctx,
+ (resource.comment.span.start, resource.comment.span.end))
+ else:
+ last_span_end = 0
+
for entry in resource.body:
if not only_localizable:
if entry.span.start > last_span_end:
yield Whitespace(
self.ctx, (last_span_end, entry.span.start))
if isinstance(entry, ftl.Message):
yield FluentEntity(self.ctx, entry)
@@ -651,19 +675,25 @@ class FluentParser(Parser):
start = entry.span.start
end = entry.span.end
# strip leading whitespace
start += re.match('\s*', entry.content).end()
# strip trailing whitespace
ws, we = re.search('\s*$', entry.content).span()
end -= we - ws
yield Junk(self.ctx, (start, end))
+ elif isinstance(entry, ftl.Comment) and not only_localizable:
+ span = (entry.span.start, entry.span.end)
+ yield Comment(self.ctx, span)
+ elif isinstance(entry, ftl.Section) and not only_localizable:
+ yield FluentSection(self.ctx, entry)
last_span_end = entry.span.end
+ # Yield Whitespace at the EOF.
if not only_localizable:
eof_offset = len(self.ctx.contents)
if eof_offset > last_span_end:
yield Whitespace(self.ctx, (last_span_end, eof_offset))
__constructors = [('\\.dtd$', DTDParser()),
('\\.properties$', PropertiesParser()),
--- a/compare_locales/tests/test_ftl.py
+++ b/compare_locales/tests/test_ftl.py
@@ -98,17 +98,17 @@ h =
abc =
A
B
C
''')
[abc] = list(self.parser)
self.assertEqual(abc.key, 'abc')
- self.assertEqual(abc.val, '\n A\n B\n C')
+ self.assertEqual(abc.val, ' A\n B\n C')
self.assertEqual(abc.all, 'abc =\n A\n B\n C')
def test_message_with_attribute(self):
self.parser.readContents('''\
abc = ABC
.attr = Attr
''')
@@ -128,57 +128,78 @@ abc
self.assertEqual(abc.val, '')
self.assertEqual(abc.all, 'abc\n .attr = Attr')
attributes = list(abc.attributes)
self.assertEqual(len(attributes), 1)
attr = attributes[0]
self.assertEqual(attr.key, 'attr')
self.assertEqual(attr.val, 'Attr')
- def test_whitespace(self):
+ def test_non_localizable(self):
self.parser.readContents('''\
// Resource Comment
foo = Foo
// Section Comment
-[[ Section ]]
+[[ Section Header ]]
bar = Bar
// Standalone Comment
// Baz Comment
baz = Baz
''')
- entities = list(self.parser.walk())
+ entities = self.parser.walk()
- self.assertTrue(isinstance(entities[0], parser.Whitespace))
- self.assertEqual(entities[0].all, '\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Comment))
+ self.assertEqual(entity.all, '// Resource Comment')
- self.assertTrue(isinstance(entities[1], parser.FluentEntity))
- self.assertEqual(entities[1].val, 'Foo')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n\n')
- self.assertTrue(isinstance(entities[2], parser.Whitespace))
- self.assertEqual(entities[2].all, '\n\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.FluentEntity))
+ self.assertEqual(entity.val, 'Foo')
+
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n\n')
- # XXX We don't yield Sections yet (bug 1399057).
-
- self.assertTrue(isinstance(entities[3], parser.Whitespace))
- self.assertEqual(entities[3].all, '\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.FluentSection))
+ self.assertEqual(
+ entity.all, '// Section Comment\n[[ Section Header ]]')
+ self.assertEqual(entity.val, 'Section Header ')
+ self.assertEqual(
+ entity.entry.comment.content, 'Section Comment')
- self.assertTrue(isinstance(entities[4], parser.FluentEntity))
- self.assertEqual(entities[4].val, 'Bar')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n\n')
- self.assertTrue(isinstance(entities[5], parser.Whitespace))
- self.assertEqual(entities[5].all, '\n\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.FluentEntity))
+ self.assertEqual(entity.val, 'Bar')
- # XXX We don't yield Comments yet (bug 1399057).
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n\n')
- self.assertTrue(isinstance(entities[6], parser.Whitespace))
- self.assertEqual(entities[6].all, '\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Comment))
+ self.assertEqual(entity.all, '// Standalone Comment')
+
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n\n')
- self.assertTrue(isinstance(entities[7], parser.FluentEntity))
- self.assertEqual(entities[7].val, 'Baz')
- self.assertEqual(entities[7].entry.comment.content, 'Baz Comment')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.FluentEntity))
+ self.assertEqual(entity.val, 'Baz')
+ self.assertEqual(entity.entry.comment.content, 'Baz Comment')
- self.assertTrue(isinstance(entities[8], parser.Whitespace))
- self.assertEqual(entities[8].all, '\n')
+ entity = next(entities)
+ self.assertTrue(isinstance(entity, parser.Whitespace))
+ self.assertEqual(entity.all, '\n')
--- a/setup.py
+++ b/setup.py
@@ -40,12 +40,12 @@ setup(name="compare-locales",
platforms=["any"],
entry_points={'console_scripts': [
'compare-locales = compare_locales.commands:CompareLocales.call']},
packages=['compare_locales', 'compare_locales.tests'],
package_data={
'compare_locales.tests': ['data/*.properties', 'data/*.dtd']
},
install_requires=[
- 'fluent>=0.4.2',
+ 'fluent==0.4.3',
'pytoml',
],
test_suite='compare_locales.tests')