--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -3,16 +3,17 @@ mozilla.pth:python/mozboot
 mozilla.pth:python/mozbuild
 mozilla.pth:python/mozlint
 mozilla.pth:python/mozversioncontrol
 mozilla.pth:third_party/python/blessings
 mozilla.pth:third_party/python/compare-locales
 mozilla.pth:third_party/python/configobj
 mozilla.pth:third_party/python/cram
 mozilla.pth:third_party/python/dlmanager
+mozilla.pth:third_party/python/fluent
 mozilla.pth:third_party/python/futures
 mozilla.pth:third_party/python/hglib
 mozilla.pth:third_party/python/jsmin
 optional:setup.py:third_party/python/psutil:build_ext:--inplace
 mozilla.pth:third_party/python/psutil
 mozilla.pth:third_party/python/pylru
 mozilla.pth:third_party/python/which
 mozilla.pth:third_party/python/pystache

--- a/third_party/python/compare-locales/compare_locales/__init__.py
+++ b/third_party/python/compare-locales/compare_locales/__init__.py
@@ -1,1 +1,1 @@
-version = "1.2.3"
+version = "2.1"

--- a/third_party/python/compare-locales/compare_locales/checks.py
+++ b/third_party/python/compare-locales/compare_locales/checks.py
@@ -1,44 +1,57 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
+from collections import Counter
 from difflib import SequenceMatcher
 from xml import sax
 try:
     from cStringIO import StringIO
 except ImportError:
     from StringIO import StringIO
 
-from compare_locales.parser import DTDParser, PropertiesParser
+from compare_locales.parser import DTDParser, PropertiesEntity
 
 
 class Checker(object):
     '''Abstract class to implement checks per file type.
     '''
     pattern = None
+    # if a check uses all reference entities, set this to True
+    needs_reference = False
 
     @classmethod
     def use(cls, file):
         return cls.pattern.match(file.file)
 
+    def __init__(self, extra_tests):
+        self.extra_tests = extra_tests
+        self.reference = None
+
     def check(self, refEnt, l10nEnt):
         '''Given the reference and localized Entities, performs checks.
 
         This is a generator yielding tuples of
         - "warning" or "error", depending on what should be reported,
         - tuple of line, column info for the error within the string
         - description string to be shown in the report
         '''
         if True:
             raise NotImplementedError("Need to subclass")
         yield ("error", (0, 0), "This is an example error", "example")
 
+    def set_reference(self, reference):
+        '''Set the reference entities.
+        Only do this if self.needs_reference is True.
+        '''
+        self.reference = reference
+
 
 class PrintfException(Exception):
     def __init__(self, msg, pos):
         self.pos = pos
         self.msg = msg
 
 
 class PropertiesChecker(Checker):
@@ -53,17 +66,18 @@ class PropertiesChecker(Checker):
 
     def check(self, refEnt, l10nEnt):
         '''Test for the different variable formats.
         '''
         refValue, l10nValue = refEnt.val, l10nEnt.val
         refSpecs = None
         # check for PluralForm.jsm stuff, should have the docs in the
         # comment
-        if 'Localization_and_Plurals' in refEnt.pre_comment:
+        if (refEnt.pre_comment
+                and 'Localization_and_Plurals' in refEnt.pre_comment.all):
             # For plurals, common variable pattern is #1. Try that.
             pats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
                                                             refValue))
             if len(pats) == 0:
                 return
             lpats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
                                                              l10nValue))
             if pats - lpats:
@@ -72,19 +86,19 @@ class PropertiesChecker(Checker):
                 return
             if lpats - pats:
                 yield ('error', 0, 'unreplaced variables in l10n',
                        'plural')
                 return
             return
         # check for lost escapes
         raw_val = l10nEnt.raw_val
-        for m in PropertiesParser.escape.finditer(raw_val):
+        for m in PropertiesEntity.escape.finditer(raw_val):
             if m.group('single') and \
-               m.group('single') not in PropertiesParser.known_escapes:
+               m.group('single') not in PropertiesEntity.known_escapes:
                 yield ('warning', m.start(),
                        'unknown escape sequence, \\' + m.group('single'),
                        'escape')
         try:
             refSpecs = self.getPrintfSpecs(refValue)
         except PrintfException:
             refSpecs = []
         if refSpecs:
@@ -154,20 +168,16 @@ class PropertiesChecker(Checker):
                 pos = int(m.group('number')) - 1
                 ls = len(specs)
                 if pos >= ls:
                     # pad specs
                     nones = pos - ls
                     specs[ls:pos] = nones*[None]
                     specs.append(m.group('spec'))
                 else:
-                    if specs[pos] is not None:
-                        raise PrintfException('Double ordered argument %d' %
-                                              (pos+1),
-                                              m.start())
                     specs[pos] = m.group('spec')
             else:
                 specs.append(m.group('spec'))
         # check for missing args
         if hasNumber and not all(specs):
             raise PrintfException('Ordered argument missing', 0)
         return specs
 
@@ -179,25 +189,29 @@ class DTDChecker(Checker):
 
     The code tries to parse until it doesn't find any unresolved entities
     anymore. If it finds one, it tries to grab the key, and adds an empty
     <!ENTITY key ""> definition to the header.
 
     Also checks for some CSS and number heuristics in the values.
     """
     pattern = re.compile('.*\.dtd$')
+    needs_reference = True  # to cast a wider net for known entity references
 
     eref = re.compile('&(%s);' % DTDParser.Name)
     tmpl = '''<!DOCTYPE elem [%s]>
 <elem>%s</elem>
 '''
     xmllist = set(('amp', 'lt', 'gt', 'apos', 'quot'))
 
-    def __init__(self, reference):
-        self.reference = reference
+    def __init__(self, extra_tests):
+        super(DTDChecker, self).__init__(extra_tests)
+        self.processContent = False
+        if self.extra_tests is not None and 'android-dtd' in self.extra_tests:
+            self.processContent = True
         self.__known_entities = None
 
     def known_entities(self, refValue):
         if self.__known_entities is None and self.reference is not None:
             self.__known_entities = set()
             for ent in self.reference:
                 self.__known_entities.update(self.entities_for_value(ent.val))
         return self.__known_entities if self.__known_entities is not None \
@@ -223,18 +237,16 @@ class DTDChecker(Checker):
     num = re.compile('^%s$' % numPattern)
     lengthPattern = '%s(em|px|ch|cm|in)' % numPattern
     length = re.compile('^%s$' % lengthPattern)
     spec = re.compile(r'((?:min\-)?(?:width|height))\s*:\s*%s' %
                       lengthPattern)
     style = re.compile(r'^%(spec)s\s*(;\s*%(spec)s\s*)*;?$' %
                        {'spec': spec.pattern})
 
-    processContent = None
-
     def check(self, refEnt, l10nEnt):
         """Try to parse the refvalue inside a dummy element, and keep
         track of entities that we need to define to make that work.
 
         Return a checker that offers just those entities.
         """
         refValue, l10nValue = refEnt.val, l10nEnt.val
         # find entities the refValue references,
@@ -258,17 +270,17 @@ class DTDChecker(Checker):
                    (0, 0),
                    "can't parse en-US value", 'xmlparse')
 
         # find entities the l10nValue references,
         # reusing markup from DTDParser.
         l10nlist = self.entities_for_value(l10nValue)
         missing = sorted(l10nlist - reflist)
         _entities = entities + ''.join('<!ENTITY %s "">' % s for s in missing)
-        if self.processContent is not None:
+        if self.processContent:
             self.texthandler.textcontent = ''
             parser.setContentHandler(self.texthandler)
         try:
             parser.parse(StringIO(self.tmpl % (_entities,
                          l10nValue.encode('utf-8'))))
             # also catch stray %
             # if this fails, we need to substract the entity definition
             parser.setContentHandler(self.defaulthandler)
@@ -342,27 +354,20 @@ class DTDChecker(Checker):
                         if u != ru:
                             msgs.append("units for %s don't match "
                                         "(%s != %s)" % (s, u, ru))
                 for s in refMap.iterkeys():
                     msgs.insert(0, '%s only in reference' % s)
                 if msgs:
                     yield ('warning', 0, ', '.join(msgs), 'css')
 
-        if self.processContent is not None:
-            for t in self.processContent(self.texthandler.textcontent):
+        if self.extra_tests is not None and 'android-dtd' in self.extra_tests:
+            for t in self.processAndroidContent(self.texthandler.textcontent):
                 yield t
 
-
-class PrincessAndroid(DTDChecker):
-    """Checker for the string values that Android puts into an XML container.
-
-    http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling  # noqa
-    has more info. Check for unescaped apostrophes and bad unicode escapes.
-    """
     quoted = re.compile("(?P<q>[\"']).*(?P=q)$")
 
     def unicode_escape(self, str):
         """Helper method to try to decode all unicode escapes in a string.
 
         This code uses the standard python decode for unicode-escape, but
         that's somewhat tricky, as its input needs to be ascii. To get to
         ascii, the unicode string gets converted to ascii with
@@ -380,25 +385,21 @@ class PrincessAndroid(DTDChecker):
         except UnicodeDecodeError, e:
             args = list(e.args)
             badstring = args[1][args[2]:args[3]]
             i = len(args[1][:args[2]].decode('unicode-escape'))
             args[2] = i
             args[3] = i + len(badstring)
             raise UnicodeDecodeError(*args)
 
-    @classmethod
-    def use(cls, file):
-        """Use this Checker only for DTD files in embedding/android."""
-        return (file.module in ("embedding/android",
-                                "mobile/android/base") and
-                cls.pattern.match(file.file))
+    def processAndroidContent(self, val):
+        """Check for the string values that Android puts into an XML container.
 
-    def processContent(self, val):
-        """Actual check code.
+        http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling  # noqa
+
         Check for unicode escapes and unescaped quotes and apostrophes,
         if string's not quoted.
         """
         # first, try to decode unicode escapes
         try:
             self.unicode_escape(val)
         except UnicodeDecodeError, e:
             yield ('error', e.args[2], e.args[4], 'android')
@@ -423,16 +424,73 @@ class PrincessAndroid(DTDChecker):
                           u"or \\u0022, or put string in apostrophes."
                 else:
                     msg = u"Apostrophes in Android DTDs need escaping with "\
                           u"\\' or \\u0027, or use \u2019, or put string in "\
                           u"quotes."
                 yield ('error', m.end(0)+offset, msg, 'android')
 
 
-def getChecker(file, reference=None):
+class FluentChecker(Checker):
+    '''Tests to run on Fluent (FTL) files.
+    '''
+    pattern = re.compile('.*\.ftl')
+
+    # Positions yielded by FluentChecker.check are absolute offsets from the
+    # beginning of the file.  This is different from the base Checker behavior
+    # which yields offsets from the beginning of the current entity's value.
+    def check(self, refEnt, l10nEnt):
+        ref_entry = refEnt.entry
+        l10n_entry = l10nEnt.entry
+        # verify that values match, either both have a value or none
+        if ref_entry.value is not None and l10n_entry.value is None:
+            yield ('error', l10n_entry.span.start,
+                   'Missing value', 'fluent')
+        if ref_entry.value is None and l10n_entry.value is not None:
+            yield ('error', l10n_entry.value.span.start,
+                   'Obsolete value', 'fluent')
+
+        # verify that we're having the same set of attributes
+        ref_attr_names = set((attr.id.name for attr in ref_entry.attributes))
+        ref_pos = dict((attr.id.name, i)
+                       for i, attr in enumerate(ref_entry.attributes))
+        l10n_attr_counts = \
+            Counter(attr.id.name for attr in l10n_entry.attributes)
+        l10n_attr_names = set(l10n_attr_counts)
+        l10n_pos = dict((attr.id.name, i)
+                        for i, attr in enumerate(l10n_entry.attributes))
+        # check for duplicate Attributes
+        # only warn to not trigger a merge skip
+        for attr_name, cnt in l10n_attr_counts.items():
+            if cnt > 1:
+                yield (
+                    'warning',
+                    l10n_entry.attributes[l10n_pos[attr_name]].span.start,
+                    'Attribute "{}" occurs {} times'.format(
+                        attr_name, cnt),
+                    'fluent')
+
+        missing_attr_names = sorted(ref_attr_names - l10n_attr_names,
+                                    key=lambda k: ref_pos[k])
+        for attr_name in missing_attr_names:
+            yield ('error', l10n_entry.span.start,
+                   'Missing attribute: ' + attr_name, 'fluent')
+
+        obsolete_attr_names = sorted(l10n_attr_names - ref_attr_names,
+                                     key=lambda k: l10n_pos[k])
+        obsolete_attrs = [
+            attr
+            for attr in l10n_entry.attributes
+            if attr.id.name in obsolete_attr_names
+        ]
+        for attr in obsolete_attrs:
+            yield ('error', attr.span.start,
+                   'Obsolete attribute: ' + attr.id.name, 'fluent')
+
+
+def getChecker(file, extra_tests=None):
     if PropertiesChecker.use(file):
-        return PropertiesChecker()
-    if PrincessAndroid.use(file):
-        return PrincessAndroid(reference)
+        return PropertiesChecker(extra_tests)
     if DTDChecker.use(file):
-        return DTDChecker(reference)
+        return DTDChecker(extra_tests)
+    if FluentChecker.use(file):
+        return FluentChecker(extra_tests)
     return None

--- a/third_party/python/compare-locales/compare_locales/commands.py
+++ b/third_party/python/compare-locales/compare_locales/commands.py
@@ -1,28 +1,30 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 'Commands exposed to commandlines'
 
 import logging
 from argparse import ArgumentParser
+import os
 
 from compare_locales import version
-from compare_locales.paths import EnumerateApp
-from compare_locales.compare import compareApp, compareDirs
-from compare_locales.webapps import compare_web_app
+from compare_locales.paths import EnumerateApp, TOMLParser, ConfigNotFound
+from compare_locales.compare import compareProjects, Observer
 
 
-class BaseCommand(object):
-    """Base class for compare-locales commands.
-    This handles command line parsing, and general sugar for setuptools
-    entry_points.
-    """
+class CompareLocales(object):
+    """Check the localization status of gecko applications.
+The first arguments are paths to the l10n.ini or toml files for the
+applications, followed by the base directory of the localization repositories.
+Then you pass in the list of locale codes you want to compare. If there are
+not locales given, the list of locales will be taken from the l10n.toml file
+or the all-locales file referenced by the application\'s l10n.ini."""
 
     def __init__(self):
         self.parser = None
 
     def get_parser(self):
         """Get an ArgumentParser, with class docstring as description.
         """
         parser = ArgumentParser(description=self.__doc__)
@@ -30,126 +32,135 @@ class BaseCommand(object):
                             version='%(prog)s ' + version)
         parser.add_argument('-v', '--verbose', action='count', dest='v',
                             default=0, help='Make more noise')
         parser.add_argument('-q', '--quiet', action='count', dest='q',
                             default=0, help='Make less noise')
         parser.add_argument('-m', '--merge',
                             help='''Use this directory to stage merged files,
 use {ab_CD} to specify a different directory for each locale''')
-        return parser
-
-    def add_data_argument(self, parser):
+        parser.add_argument('config_paths', metavar='l10n.toml', nargs='+',
+                            help='TOML or INI file for the project')
+        parser.add_argument('l10n_base_dir', metavar='l10n-base-dir',
+                            help='Parent directory of localizations')
+        parser.add_argument('locales', nargs='*', metavar='locale-code',
+                            help='Locale code and top-level directory of '
+                                 'each localization')
+        parser.add_argument('-D', action='append', metavar='var=value',
+                            default=[], dest='defines',
+                            help='Overwrite variables in TOML files')
+        parser.add_argument('--unified', action="store_true",
+                            help="Show output for all projects unified")
+        parser.add_argument('--full', action="store_true",
+                            help="Compare projects that are disabled")
+        parser.add_argument('--clobber-merge', action="store_true",
+                            default=False, dest='clobber',
+                            help="""WARNING: DATALOSS.
+Use this option with care. If specified, the merge directory will
+be clobbered for each module. That means, the subdirectory will
+be completely removed, any files that were there are lost.
+Be careful to specify the right merge directory when using this option.""")
         parser.add_argument('--data', choices=['text', 'exhibit', 'json'],
                             default='text',
                             help='''Choose data and format (one of text,
 exhibit, json); text: (default) Show which files miss which strings, together
 with warnings and errors. Also prints a summary; json: Serialize the internal
 tree, useful for tools. Also always succeeds; exhibit: Serialize the summary
 data in a json useful for Exhibit
 ''')
+        return parser
 
     @classmethod
     def call(cls):
         """Entry_point for setuptools.
         The actual command handling is done in the handle() method of the
         subclasses.
         """
         cmd = cls()
-        cmd.handle_()
+        return cmd.handle_()
 
     def handle_(self):
         """The instance part of the classmethod call."""
         self.parser = self.get_parser()
         args = self.parser.parse_args()
         # log as verbose or quiet as we want, warn by default
         logging.basicConfig()
         logging.getLogger().setLevel(logging.WARNING -
                                      (args.v - args.q) * 10)
-        observer = self.handle(args)
-        print observer.serialize(type=args.data).encode('utf-8', 'replace')
-
-    def handle(self, args):
-        """Subclasses need to implement this method for the actual
-        command handling.
-        """
-        raise NotImplementedError
-
-
-class CompareLocales(BaseCommand):
-    """Check the localization status of a gecko application.
-The first argument is a path to the l10n.ini file for the application,
-followed by the base directory of the localization repositories.
-Then you pass in the list of locale codes you want to compare. If there are
-not locales given, the list of locales will be taken from the all-locales file
-of the application\'s l10n.ini."""
+        kwargs = vars(args)
+        # strip handeld arguments
+        kwargs.pop('q')
+        kwargs.pop('v')
+        return self.handle(**kwargs)
 
-    def get_parser(self):
-        parser = super(CompareLocales, self).get_parser()
-        parser.add_argument('ini_file', metavar='l10n.ini',
-                            help='INI file for the project')
-        parser.add_argument('l10n_base_dir', metavar='l10n-base-dir',
-                            help='Parent directory of localizations')
-        parser.add_argument('locales', nargs='*', metavar='locale-code',
-                            help='Locale code and top-level directory of '
-                                 'each localization')
-        parser.add_argument('--clobber-merge', action="store_true",
-                            default=False, dest='clobber',
-                            help="""WARNING: DATALOSS.
-Use this option with care. If specified, the merge directory will
-be clobbered for each module. That means, the subdirectory will
-be completely removed, any files that were there are lost.
-Be careful to specify the right merge directory when using this option.""")
-        parser.add_argument('-r', '--reference', default='en-US',
-                            dest='reference',
-                            help='Explicitly set the reference '
-                            'localization. [default: en-US]')
-        self.add_data_argument(parser)
-        return parser
-
-    def handle(self, args):
-        app = EnumerateApp(args.ini_file, args.l10n_base_dir, args.locales)
-        app.reference = args.reference
+    def handle(self, config_paths, l10n_base_dir, locales,
+               merge=None, defines=None, unified=False, full=False,
+               clobber=False, data='text'):
+        # using nargs multiple times in argparser totally screws things
+        # up, repair that.
+        # First files are configs, then the base dir, everything else is
+        # locales
+        all_args = config_paths + [l10n_base_dir] + locales
+        config_paths = []
+        locales = []
+        if defines is None:
+            defines = []
+        while all_args and not os.path.isdir(all_args[0]):
+            config_paths.append(all_args.pop(0))
+        if not config_paths:
+            self.parser.error('no configuration file given')
+        for cf in config_paths:
+            if not os.path.isfile(cf):
+                self.parser.error('config file %s not found' % cf)
+        if not all_args:
+            self.parser.error('l10n-base-dir not found')
+        l10n_base_dir = all_args.pop(0)
+        locales.extend(all_args)
+        # when we compare disabled projects, we set our locales
+        # on all subconfigs, so deep is True.
+        locales_deep = full
+        configs = []
+        config_env = {}
+        for define in defines:
+            var, _, value = define.partition('=')
+            config_env[var] = value
+        for config_path in config_paths:
+            if config_path.endswith('.toml'):
+                try:
+                    config = TOMLParser.parse(config_path, env=config_env)
+                except ConfigNotFound as e:
+                    self.parser.exit('config file %s not found' % e.filename)
+                config.add_global_environment(l10n_base=l10n_base_dir)
+                if locales:
+                    config.set_locales(locales, deep=locales_deep)
+                configs.append(config)
+            else:
+                app = EnumerateApp(
+                    config_path, l10n_base_dir, locales)
+                configs.append(app.asConfig())
         try:
-            observer = compareApp(app, merge_stage=args.merge,
-                                  clobber=args.clobber)
+            unified_observer = None
+            if unified:
+                unified_observer = Observer()
+            observers = compareProjects(
+                configs,
+                stat_observer=unified_observer,
+                merge_stage=merge, clobber_merge=clobber)
         except (OSError, IOError), exc:
             print "FAIL: " + str(exc)
             self.parser.exit(2)
-        return observer
-
-
-class CompareDirs(BaseCommand):
-    """Check the localization status of a directory tree.
-The first argument is a path to the reference data,the second is the
-localization to be tested."""
-
-    def get_parser(self):
-        parser = super(CompareDirs, self).get_parser()
-        parser.add_argument('reference')
-        parser.add_argument('localization')
-        self.add_data_argument(parser)
-        return parser
-
-    def handle(self, args):
-        observer = compareDirs(args.reference, args.localization,
-                               merge_stage=args.merge)
-        return observer
+        if unified:
+            observers = [unified_observer]
 
-
-class CompareWebApp(BaseCommand):
-    """Check the localization status of a gaia-style web app.
-The first argument is the directory of the web app.
-Following arguments explicitly state the locales to test.
-If none are given, test all locales in manifest.webapp or files."""
-
-    def get_parser(self):
-        parser = super(CompareWebApp, self).get_parser()
-        parser.add_argument('webapp')
-        parser.add_argument('locales', nargs='*', metavar='locale-code',
-                            help='Locale code and top-level directory of '
-                                 'each localization')
-        self.add_data_argument(parser)
-        return parser
-
-    def handle(self, args):
-        observer = compare_web_app(args.webapp, args.locales)
-        return observer
+        rv = 0
+        for observer in observers:
+            print observer.serialize(type=data).encode('utf-8', 'replace')
+            # summary is a dict of lang-summary dicts
+            # find out if any of our results has errors, return 1 if so
+            if rv > 0:
+                continue  # we already have errors
+            for loc, summary in observer.summary.items():
+                if summary.get('errors', 0) > 0:
+                    rv = 1
+                    # no need to check further summaries, but
+                    # continue to run through observers
+                    break
+        return rv

--- a/third_party/python/compare-locales/compare_locales/compare.py
+++ b/third_party/python/compare-locales/compare_locales/compare.py
@@ -1,43 +1,43 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 'Mozilla l10n compare locales tool'
 
 import codecs
 import os
-import os.path
 import shutil
 import re
-from difflib import SequenceMatcher
 from collections import defaultdict
 
 try:
     from json import dumps
 except:
     from simplejson import dumps
 
 from compare_locales import parser
-from compare_locales import paths
+from compare_locales import paths, mozpath
 from compare_locales.checks import getChecker
 
 
 class Tree(object):
     def __init__(self, valuetype):
         self.branches = dict()
         self.valuetype = valuetype
         self.value = None
 
     def __getitem__(self, leaf):
         parts = []
         if isinstance(leaf, paths.File):
-            parts = [p for p in [leaf.locale, leaf.module] if p] + \
-                leaf.file.split('/')
+            parts = [] if not leaf.locale else [leaf.locale]
+            if leaf.module:
+                parts += leaf.module.split('/')
+            parts += leaf.file.split('/')
         else:
             parts = leaf.split('/')
         return self.__get(parts)
 
     def __get(self, parts):
         common = None
         old = None
         new = tuple(parts)
@@ -88,180 +88,156 @@ class Tree(object):
             for child in self.branches[key].getContent(depth + 1):
                 yield child
 
     def toJSON(self):
         '''
         Returns this Tree as a JSON-able tree of hashes.
         Only the values need to take care that they're JSON-able.
         '''
-        json = {}
-        keys = self.branches.keys()
-        keys.sort()
         if self.value is not None:
-            json['value'] = self.value
-        children = [('/'.join(key), self.branches[key].toJSON())
-                    for key in keys]
-        if children:
-            json['children'] = children
-        return json
+            return self.value
+        return dict(('/'.join(key), self.branches[key].toJSON())
+                    for key in self.branches.keys())
 
     def getStrRows(self):
         def tostr(t):
             if t[1] == 'key':
                 return self.indent * t[0] + '/'.join(t[2])
             return self.indent * (t[0] + 1) + str(t[2])
 
         return map(tostr, self.getContent())
 
     def __str__(self):
         return '\n'.join(self.getStrRows())
 
 
-class AddRemove(SequenceMatcher):
+class AddRemove(object):
     def __init__(self):
-        SequenceMatcher.__init__(self, None, None, None)
+        self.left = self.right = None
 
     def set_left(self, left):
         if not isinstance(left, list):
-            left = [l for l in left]
-        self.set_seq1(left)
+            left = list(l for l in left)
+        self.left = left
 
     def set_right(self, right):
         if not isinstance(right, list):
-            right = [l for l in right]
-        self.set_seq2(right)
+            right = list(l for l in right)
+        self.right = right
 
     def __iter__(self):
-        for tag, i1, i2, j1, j2 in self.get_opcodes():
-            if tag == 'equal':
-                for pair in zip(self.a[i1:i2], self.b[j1:j2]):
-                    yield ('equal', pair)
-            elif tag == 'delete':
-                for item in self.a[i1:i2]:
-                    yield ('delete', item)
-            elif tag == 'insert':
-                for item in self.b[j1:j2]:
-                    yield ('add', item)
+        # order_map stores index in left and then index in right
+        order_map = dict((item, (i, -1)) for i, item in enumerate(self.left))
+        left_items = set(order_map)
+        # as we go through the right side, keep track of which left
+        # item we had in right last, and for items not in left,
+        # set the sortmap to (left_offset, right_index)
+        left_offset = -1
+        right_items = set()
+        for i, item in enumerate(self.right):
+            right_items.add(item)
+            if item in order_map:
+                left_offset = order_map[item][0]
             else:
-                # tag == 'replace'
-                for item in self.a[i1:i2]:
-                    yield ('delete', item)
-                for item in self.b[j1:j2]:
-                    yield ('add', item)
-
-
-class DirectoryCompare(SequenceMatcher):
-    def __init__(self, reference):
-        SequenceMatcher.__init__(self, None, [i for i in reference],
-                                 [])
-        self.watcher = None
-
-    def setWatcher(self, watcher):
-        self.watcher = watcher
-
-    def compareWith(self, other):
-        if not self.watcher:
-            return
-        self.set_seq2([i for i in other])
-        for tag, i1, i2, j1, j2 in self.get_opcodes():
-            if tag == 'equal':
-                for i, j in zip(xrange(i1, i2), xrange(j1, j2)):
-                    self.watcher.compare(self.a[i], self.b[j])
-            elif tag == 'delete':
-                for i in xrange(i1, i2):
-                    self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
-            elif tag == 'insert':
-                for j in xrange(j1, j2):
-                    self.watcher.remove(self.b[j])
+                order_map[item] = (left_offset, i)
+        for item in sorted(order_map, key=lambda item: order_map[item]):
+            if item in left_items and item in right_items:
+                yield ('equal', item)
+            elif item in left_items:
+                yield ('delete', item)
             else:
-                for j in xrange(j1, j2):
-                    self.watcher.remove(self.b[j])
-                for i in xrange(i1, i2):
-                    self.watcher.add(self.a[i], other.cloneFile(self.a[i]))
+                yield ('add', item)
 
 
 class Observer(object):
-    stat_cats = ['missing', 'obsolete', 'missingInFiles', 'report',
-                 'changed', 'unchanged', 'keys']
 
-    def __init__(self):
-        class intdict(defaultdict):
-            def __init__(self):
-                defaultdict.__init__(self, int)
-
-        self.summary = defaultdict(intdict)
-        self.details = Tree(dict)
-        self.filter = None
+    def __init__(self, filter=None, file_stats=False):
+        self.summary = defaultdict(lambda: defaultdict(int))
+        self.details = Tree(list)
+        self.filter = filter
+        self.file_stats = None
+        if file_stats:
+            self.file_stats = defaultdict(lambda: defaultdict(dict))
 
     # support pickling
     def __getstate__(self):
-        return dict(summary=self.getSummary(), details=self.details)
+        state = dict(summary=self._dictify(self.summary), details=self.details)
+        if self.file_stats is not None:
+            state['file_stats'] = self._dictify(self.file_stats)
+        return state
 
     def __setstate__(self, state):
-        class intdict(defaultdict):
-            def __init__(self):
-                defaultdict.__init__(self, int)
-
-        self.summary = defaultdict(intdict)
+        self.summary = defaultdict(lambda: defaultdict(int))
         if 'summary' in state:
             for loc, stats in state['summary'].iteritems():
                 self.summary[loc].update(stats)
+        self.file_stats = None
+        if 'file_stats' in state:
+            self.file_stats = defaultdict(lambda: defaultdict(dict))
+            for k, d in state['file_stats'].iteritems():
+                self.file_stats[k].update(d)
         self.details = state['details']
         self.filter = None
 
-    def getSummary(self):
+    def _dictify(self, d):
         plaindict = {}
-        for k, v in self.summary.iteritems():
+        for k, v in d.iteritems():
             plaindict[k] = dict(v)
         return plaindict
 
     def toJSON(self):
-        return dict(summary=self.getSummary(), details=self.details.toJSON())
+        # Don't export file stats, even if we collected them.
+        # Those are not part of the data we use toJSON for.
+        return {
+            'summary': self._dictify(self.summary),
+            'details': self.details.toJSON()
+        }
+
+    def updateStats(self, file, stats):
+        # in multi-project scenarios, this file might not be ours,
+        # check that.
+        # Pass in a dummy entity key '' to avoid getting in to
+        # generic file filters. If we have stats for those,
+        # we want to aggregate the counts
+        if (self.filter is not None and
+                self.filter(file, entity='') == 'ignore'):
+            return
+        for category, value in stats.iteritems():
+            self.summary[file.locale][category] += value
+        if self.file_stats is None:
+            return
+        if 'missingInFiles' in stats:
+            # keep track of how many strings are in a missing file
+            # we got the {'missingFile': 'error'} from the notify pass
+            self.details[file].append({'count': stats['missingInFiles']})
+            # missingInFiles should just be "missing" in file stats
+            self.file_stats[file.locale][file.localpath]['missing'] = \
+                stats['missingInFiles']
+            return  # there are no other stats for missing files
+        self.file_stats[file.locale][file.localpath].update(stats)
 
     def notify(self, category, file, data):
-        rv = "error"
-        if category in self.stat_cats:
-            # these get called post reporting just for stats
-            # return "error" to forward them to other other_observers
-            self.summary[file.locale][category] += data
-            # keep track of how many strings are in a missing file
-            # we got the {'missingFile': 'error'} from the first pass
-            if category == 'missingInFiles':
-                self.details[file]['strings'] = data
-            return "error"
+        rv = 'error'
         if category in ['missingFile', 'obsoleteFile']:
             if self.filter is not None:
                 rv = self.filter(file)
             if rv != "ignore":
-                self.details[file][category] = rv
+                self.details[file].append({category: rv})
             return rv
         if category in ['missingEntity', 'obsoleteEntity']:
             if self.filter is not None:
                 rv = self.filter(file, data)
             if rv == "ignore":
                 return rv
-            v = self.details[file]
-            try:
-                v[category].append(data)
-            except KeyError:
-                v[category] = [data]
+            self.details[file].append({category: data})
             return rv
-        if category == 'error':
-            try:
-                self.details[file][category].append(data)
-            except KeyError:
-                self.details[file][category] = [data]
-            self.summary[file.locale]['errors'] += 1
-        elif category == 'warning':
-            try:
-                self.details[file][category].append(data)
-            except KeyError:
-                self.details[file][category] = [data]
-            self.summary[file.locale]['warnings'] += 1
+        if category in ('error', 'warning'):
+            self.details[file].append({category: data})
+            self.summary[file.locale][category + 's'] += 1
         return rv
 
     def toExhibit(self):
         items = []
         for locale in sorted(self.summary.iterkeys()):
             summary = self.summary[locale]
             if locale is not None:
                 item = {'id': 'xxx/' + locale,
@@ -271,37 +247,44 @@ class Observer(object):
                 item = {'id': 'xxx',
                         'label': 'xxx',
                         'locale': 'xxx'}
             item['type'] = 'Build'
             total = sum([summary[k]
                          for k in ('changed', 'unchanged', 'report', 'missing',
                                    'missingInFiles')
                          if k in summary])
+            total_w = sum([summary[k]
+                           for k in ('changed_w', 'unchanged_w', 'missing_w')
+                           if k in summary])
             rate = (('changed' in summary and summary['changed'] * 100) or
                     0) / total
             item.update((k, summary.get(k, 0))
                         for k in ('changed', 'unchanged'))
             item.update((k, summary[k])
                         for k in ('report', 'errors', 'warnings')
                         if k in summary)
             item['missing'] = summary.get('missing', 0) + \
                 summary.get('missingInFiles', 0)
             item['completion'] = rate
             item['total'] = total
+            item.update((k, summary.get(k, 0))
+                        for k in ('changed_w', 'unchanged_w', 'missing_w'))
+            item['total_w'] = total_w
             result = 'success'
             if item.get('warnings', 0):
                 result = 'warning'
             if item.get('errors', 0) or item.get('missing', 0):
                 result = 'failure'
             item['result'] = result
             items.append(item)
         data = {
             "properties": dict.fromkeys(
                 ("completion", "errors", "warnings", "missing", "report",
+                 "missing_w", "changed_w", "unchanged_w",
                  "unchanged", "changed", "obsolete"),
                 {"valueType": "number"}),
             "types": {
                 "Build": {"pluralLabel": "Builds"}
             }}
         data['items'] = items
         return dumps(data, indent=2)
 
@@ -311,36 +294,29 @@ class Observer(object):
         if type == "json":
             return dumps(self.toJSON())
 
         def tostr(t):
             if t[1] == 'key':
                 return '  ' * t[0] + '/'.join(t[2])
             o = []
             indent = '  ' * (t[0] + 1)
-            if 'error' in t[2]:
-                o += [indent + 'ERROR: ' + e for e in t[2]['error']]
-            if 'warning' in t[2]:
-                o += [indent + 'WARNING: ' + e for e in t[2]['warning']]
-            if 'missingEntity' in t[2] or 'obsoleteEntity' in t[2]:
-                missingEntities = ('missingEntity' in t[2] and
-                                   t[2]['missingEntity']) or []
-                obsoleteEntities = ('obsoleteEntity' in t[2] and
-                                    t[2]['obsoleteEntity']) or []
-                entities = missingEntities + obsoleteEntities
-                entities.sort()
-                for entity in entities:
-                    op = '+'
-                    if entity in obsoleteEntities:
-                        op = '-'
-                    o.append(indent + op + entity)
-            elif 'missingFile' in t[2]:
-                o.append(indent + '// add and localize this file')
-            elif 'obsoleteFile' in t[2]:
-                o.append(indent + '// remove this file')
+            for item in t[2]:
+                if 'error' in item:
+                    o += [indent + 'ERROR: ' + item['error']]
+                elif 'warning' in item:
+                    o += [indent + 'WARNING: ' + item['warning']]
+                elif 'missingEntity' in item:
+                    o += [indent + '+' + item['missingEntity']]
+                elif 'obsoleteEntity' in item:
+                    o += [indent + '-' + item['obsoleteEntity']]
+                elif 'missingFile' in item:
+                    o.append(indent + '// add and localize this file')
+                elif 'obsoleteFile' in item:
+                    o.append(indent + '// remove this file')
             return '\n'.join(o)
 
         out = []
         for locale, summary in sorted(self.summary.iteritems()):
             if locale is not None:
                 out.append(locale + ':')
             out += [k + ': ' + str(v) for k, v in sorted(summary.iteritems())]
             total = sum([summary[k]
@@ -357,276 +333,310 @@ class Observer(object):
     def __str__(self):
         return 'observer'
 
 
 class ContentComparer:
     keyRE = re.compile('[kK]ey')
     nl = re.compile('\n', re.M)
 
-    def __init__(self):
+    def __init__(self, observers, stat_observers=None):
         '''Create a ContentComparer.
         observer is usually a instance of Observer. The return values
         of the notify method are used to control the handling of missing
         entities.
         '''
-        self.reference = dict()
-        self.observer = Observer()
-        self.other_observers = []
-        self.merge_stage = None
+        self.observers = observers
+        if stat_observers is None:
+            stat_observers = []
+        self.stat_observers = stat_observers
 
-    def add_observer(self, obs):
-        '''Add a non-filtering observer.
-        Results from the notify calls are ignored.
-        '''
-        self.other_observers.append(obs)
-
-    def set_merge_stage(self, merge_stage):
-        self.merge_stage = merge_stage
-
-    def merge(self, ref_entities, ref_map, ref_file, l10n_file, missing,
-              skips, ctx, canMerge, encoding):
-        outfile = os.path.join(self.merge_stage, l10n_file.module,
-                               l10n_file.file)
-        outdir = os.path.dirname(outfile)
+    def create_merge_dir(self, merge_file):
+        outdir = mozpath.dirname(merge_file)
         if not os.path.isdir(outdir):
             os.makedirs(outdir)
-        if not canMerge:
-            shutil.copyfile(ref_file.fullpath, outfile)
-            print "copied reference to " + outfile
+
+    def merge(self, ref_entities, ref_map, ref_file, l10n_file, merge_file,
+              missing, skips, ctx, capabilities, encoding):
+
+        if capabilities == parser.CAN_NONE:
             return
+
+        if capabilities & parser.CAN_COPY and (skips or missing):
+            self.create_merge_dir(merge_file)
+            shutil.copyfile(ref_file.fullpath, merge_file)
+            print "copied reference to " + merge_file
+            return
+
+        if not (capabilities & parser.CAN_SKIP):
+            return
+
+        # Start with None in case the merge file doesn't need to be created.
+        f = None
+
         if skips:
             # skips come in ordered by key name, we need them in file order
             skips.sort(key=lambda s: s.span[0])
-        trailing = (['\n'] +
-                    [ref_entities[ref_map[key]].all for key in missing] +
-                    [ref_entities[ref_map[skip.key]].all for skip in skips
-                     if not isinstance(skip, parser.Junk)])
-        if skips:
-            # we need to skip a few errornous blocks in the input, copy by hand
-            f = codecs.open(outfile, 'wb', encoding)
+
+            # we need to skip a few erroneous blocks in the input, copy by hand
+            self.create_merge_dir(merge_file)
+            f = codecs.open(merge_file, 'wb', encoding)
             offset = 0
             for skip in skips:
                 chunk = skip.span
                 f.write(ctx.contents[offset:chunk[0]])
                 offset = chunk[1]
             f.write(ctx.contents[offset:])
-        else:
-            shutil.copyfile(l10n_file.fullpath, outfile)
-            f = codecs.open(outfile, 'ab', encoding)
-        print "adding to " + outfile
+
+        if not (capabilities & parser.CAN_MERGE):
+            return
+
+        if skips or missing:
+            if f is None:
+                self.create_merge_dir(merge_file)
+                shutil.copyfile(l10n_file.fullpath, merge_file)
+                f = codecs.open(merge_file, 'ab', encoding)
 
-        def ensureNewline(s):
-            if not s.endswith('\n'):
-                return s + '\n'
-            return s
+            trailing = (['\n'] +
+                        [ref_entities[ref_map[key]].all for key in missing] +
+                        [ref_entities[ref_map[skip.key]].all for skip in skips
+                         if not isinstance(skip, parser.Junk)])
 
-        f.write(''.join(map(ensureNewline, trailing)))
-        f.close()
+            def ensureNewline(s):
+                if not s.endswith('\n'):
+                    return s + '\n'
+                return s
+
+            print "adding to " + merge_file
+            f.write(''.join(map(ensureNewline, trailing)))
+
+        if f is not None:
+            f.close()
 
     def notify(self, category, file, data):
         """Check observer for the found data, and if it's
-        not to ignore, notify other_observers.
+        not to ignore, notify stat_observers.
         """
-        rv = self.observer.notify(category, file, data)
-        if rv == 'ignore':
-            return rv
-        for obs in self.other_observers:
-            # non-filtering other_observers, ignore results
+        rvs = set(
+            observer.notify(category, file, data)
+            for observer in self.observers
+            )
+        if all(rv == 'ignore' for rv in rvs):
+            return 'ignore'
+        rvs.discard('ignore')
+        for obs in self.stat_observers:
+            # non-filtering stat_observers, ignore results
             obs.notify(category, file, data)
-        return rv
+        if 'error' in rvs:
+            return 'error'
+        assert len(rvs) == 1
+        return rvs.pop()
+
+    def updateStats(self, file, stats):
+        """Check observer for the found data, and if it's
+        not to ignore, notify stat_observers.
+        """
+        for observer in self.observers + self.stat_observers:
+            observer.updateStats(file, stats)
 
     def remove(self, obsolete):
         self.notify('obsoleteFile', obsolete, None)
         pass
 
-    def compare(self, ref_file, l10n):
+    def compare(self, ref_file, l10n, merge_file, extra_tests=None):
         try:
             p = parser.getParser(ref_file.file)
         except UserWarning:
             # no comparison, XXX report?
             return
-        if ref_file not in self.reference:
-            # we didn't parse this before
-            try:
-                p.readContents(ref_file.getContents())
-            except Exception, e:
-                self.notify('error', ref_file, str(e))
-                return
-            self.reference[ref_file] = p.parse()
-        ref = self.reference[ref_file]
-        ref_list = ref[1].keys()
-        ref_list.sort()
+        try:
+            p.readContents(ref_file.getContents())
+        except Exception, e:
+            self.notify('error', ref_file, str(e))
+            return
+        ref_entities, ref_map = p.parse()
         try:
             p.readContents(l10n.getContents())
             l10n_entities, l10n_map = p.parse()
             l10n_ctx = p.ctx
         except Exception, e:
             self.notify('error', l10n, str(e))
             return
 
-        l10n_list = l10n_map.keys()
-        l10n_list.sort()
         ar = AddRemove()
-        ar.set_left(ref_list)
-        ar.set_right(l10n_list)
+        ar.set_left(e.key for e in ref_entities)
+        ar.set_right(e.key for e in l10n_entities)
         report = missing = obsolete = changed = unchanged = keys = 0
+        missing_w = changed_w = unchanged_w = 0  # word stats
         missings = []
         skips = []
-        checker = getChecker(l10n, reference=ref[0])
-        for action, item_or_pair in ar:
+        checker = getChecker(l10n, extra_tests=extra_tests)
+        if checker and checker.needs_reference:
+            checker.set_reference(ref_entities)
+        for msg in p.findDuplicates(ref_entities):
+            self.notify('warning', l10n, msg)
+        for msg in p.findDuplicates(l10n_entities):
+            self.notify('error', l10n, msg)
+        for action, entity_id in ar:
             if action == 'delete':
                 # missing entity
-                _rv = self.notify('missingEntity', l10n, item_or_pair)
+                if isinstance(ref_entities[ref_map[entity_id]], parser.Junk):
+                    self.notify('warning', l10n, 'Parser error in en-US')
+                    continue
+                _rv = self.notify('missingEntity', l10n, entity_id)
                 if _rv == "ignore":
                     continue
                 if _rv == "error":
                     # only add to missing entities for l10n-merge on error,
                     # not report
-                    missings.append(item_or_pair)
+                    missings.append(entity_id)
                     missing += 1
+                    refent = ref_entities[ref_map[entity_id]]
+                    missing_w += refent.count_words()
                 else:
                     # just report
                     report += 1
             elif action == 'add':
                 # obsolete entity or junk
-                if isinstance(l10n_entities[l10n_map[item_or_pair]],
+                if isinstance(l10n_entities[l10n_map[entity_id]],
                               parser.Junk):
-                    junk = l10n_entities[l10n_map[item_or_pair]]
+                    junk = l10n_entities[l10n_map[entity_id]]
                     params = (junk.val,) + junk.position() + junk.position(-1)
                     self.notify('error', l10n,
-                                'Unparsed content "%s" from line %d colum %d'
+                                'Unparsed content "%s" from line %d column %d'
                                 ' to line %d column %d' % params)
-                    if self.merge_stage is not None:
+                    if merge_file is not None:
                         skips.append(junk)
                 elif self.notify('obsoleteEntity', l10n,
-                                 item_or_pair) != 'ignore':
+                                 entity_id) != 'ignore':
                     obsolete += 1
             else:
                 # entity found in both ref and l10n, check for changed
-                entity = item_or_pair[0]
-                refent = ref[0][ref[1][entity]]
-                l10nent = l10n_entities[l10n_map[entity]]
-                if self.keyRE.search(entity):
+                refent = ref_entities[ref_map[entity_id]]
+                l10nent = l10n_entities[l10n_map[entity_id]]
+                if self.keyRE.search(entity_id):
                     keys += 1
                 else:
-                    if refent.val == l10nent.val:
+                    if refent.equals(l10nent):
                         self.doUnchanged(l10nent)
                         unchanged += 1
+                        unchanged_w += refent.count_words()
                     else:
                         self.doChanged(ref_file, refent, l10nent)
                         changed += 1
+                        changed_w += refent.count_words()
                         # run checks:
                 if checker:
                     for tp, pos, msg, cat in checker.check(refent, l10nent):
-                        # compute real src position, if first line,
-                        # col needs adjustment
-                        if isinstance(pos, tuple):
-                            _l, col = l10nent.value_position()
-                            # line, column
-                            if pos[0] == 1:
-                                col = col + pos[1]
-                            else:
-                                col = pos[1]
-                                _l += pos[0] - 1
-                        else:
-                            _l, col = l10nent.value_position(pos)
+                        line, col = l10nent.value_position(pos)
                         # skip error entities when merging
-                        if tp == 'error' and self.merge_stage is not None:
+                        if tp == 'error' and merge_file is not None:
                             skips.append(l10nent)
                         self.notify(tp, l10n,
                                     u"%s at line %d, column %d for %s" %
-                                    (msg, _l, col, refent.key))
+                                    (msg, line, col, refent.key))
                 pass
-        if missing:
-            self.notify('missing', l10n, missing)
-        if self.merge_stage is not None and (missings or skips):
+
+        if merge_file is not None:
             self.merge(
-                ref[0], ref[1], ref_file,
-                l10n, missings, skips, l10n_ctx,
-                p.canMerge, p.encoding)
-        if report:
-            self.notify('report', l10n, report)
-        if obsolete:
-            self.notify('obsolete', l10n, obsolete)
-        if changed:
-            self.notify('changed', l10n, changed)
-        if unchanged:
-            self.notify('unchanged', l10n, unchanged)
-        if keys:
-            self.notify('keys', l10n, keys)
+                ref_entities, ref_map, ref_file,
+                l10n, merge_file, missings, skips, l10n_ctx,
+                p.capabilities, p.encoding)
+
+        stats = {}
+        for cat, value in (
+                ('missing', missing),
+                ('missing_w', missing_w),
+                ('report', report),
+                ('obsolete', obsolete),
+                ('changed', changed),
+                ('changed_w', changed_w),
+                ('unchanged', unchanged),
+                ('unchanged_w', unchanged_w),
+                ('keys', keys)):
+            if value:
+                stats[cat] = value
+        self.updateStats(l10n, stats)
         pass
 
     def add(self, orig, missing):
         if self.notify('missingFile', missing, None) == "ignore":
             # filter said that we don't need this file, don't count it
             return
         f = orig
         try:
             p = parser.getParser(f.file)
         except UserWarning:
             return
         try:
             p.readContents(f.getContents())
             entities, map = p.parse()
-        except Exception, e:
-            self.notify('error', f, str(e))
+        except Exception, ex:
+            self.notify('error', f, str(ex))
             return
-        self.notify('missingInFiles', missing, len(map))
+        # strip parse errors
+        entities = [e for e in entities if not isinstance(e, parser.Junk)]
+        self.updateStats(missing, {'missingInFiles': len(entities)})
+        missing_w = 0
+        for e in entities:
+            missing_w += e.count_words()
+        self.updateStats(missing, {'missing_w': missing_w})
 
     def doUnchanged(self, entity):
         # overload this if needed
         pass
 
     def doChanged(self, file, ref_entity, l10n_entity):
         # overload this if needed
         pass
 
 
-def compareApp(app, other_observer=None, merge_stage=None, clobber=False):
-    '''Compare locales set in app.
-
-    Optional arguments are:
-    - other_observer. A object implementing
-        notify(category, _file, data)
-      The return values of that callback are ignored.
-    - merge_stage. A directory to be used for staging the output of
-      l10n-merge.
-    - clobber. Clobber the module subdirectories of the merge dir as we go.
-      Use wisely, as it might cause data loss.
-    '''
-    comparer = ContentComparer()
-    if other_observer is not None:
-        comparer.add_observer(other_observer)
-    comparer.observer.filter = app.filter
-    for module, reference, locales in app:
-        dir_comp = DirectoryCompare(reference)
-        dir_comp.setWatcher(comparer)
-        for _, localization in locales:
-            if merge_stage is not None:
-                locale_merge = merge_stage.format(ab_CD=localization.locale)
-                comparer.set_merge_stage(locale_merge)
-                if clobber:
-                    # if clobber, remove the stage for the module if it exists
-                    clobberdir = os.path.join(locale_merge, module)
+def compareProjects(project_configs, stat_observer=None,
+                    file_stats=False,
+                    merge_stage=None, clobber_merge=False):
+    locales = set()
+    observers = []
+    for project in project_configs:
+        observers.append(
+            Observer(filter=project.filter, file_stats=file_stats))
+        locales.update(project.locales)
+    if stat_observer is not None:
+        stat_observers = [stat_observer]
+    else:
+        stat_observers = None
+    comparer = ContentComparer(observers, stat_observers=stat_observers)
+    for locale in sorted(locales):
+        files = paths.ProjectFiles(locale, project_configs,
+                                   mergebase=merge_stage)
+        root = mozpath.commonprefix([m['l10n'].prefix for m in files.matchers])
+        if merge_stage is not None:
+            if clobber_merge:
+                mergematchers = set(_m.get('merge') for _m in files.matchers)
+                mergematchers.discard(None)
+                for matcher in mergematchers:
+                    clobberdir = matcher.prefix
                     if os.path.exists(clobberdir):
                         shutil.rmtree(clobberdir)
                         print "clobbered " + clobberdir
-            dir_comp.compareWith(localization)
-    return comparer.observer
-
-
-def compareDirs(reference, locale, other_observer=None, merge_stage=None):
-    '''Compare reference and locale dir.
-
-    Optional arguments are:
-    - other_observer. A object implementing
-        notify(category, _file, data)
-      The return values of that callback are ignored.
-    '''
-    comparer = ContentComparer()
-    if other_observer is not None:
-        comparer.add_observer(other_observer)
-    comparer.set_merge_stage(merge_stage)
-    dir_comp = DirectoryCompare(paths.EnumerateDir(reference))
-    dir_comp.setWatcher(comparer)
-    dir_comp.compareWith(paths.EnumerateDir(locale))
-    return comparer.observer
+        for l10npath, refpath, mergepath, extra_tests in files:
+            # module and file path are needed for legacy filter.py support
+            module = None
+            fpath = mozpath.relpath(l10npath, root)
+            for _m in files.matchers:
+                if _m['l10n'].match(l10npath):
+                    if _m['module']:
+                        # legacy ini support, set module, and resolve
+                        # local path against the matcher prefix,
+                        # which includes the module
+                        module = _m['module']
+                        fpath = mozpath.relpath(l10npath, _m['l10n'].prefix)
+                    break
+            reffile = paths.File(refpath, fpath or refpath, module=module)
+            l10n = paths.File(l10npath, fpath or l10npath,
+                              module=module, locale=locale)
+            if not os.path.exists(l10npath):
+                comparer.add(reffile, l10n)
+                continue
+            if not os.path.exists(refpath):
+                comparer.remove(l10n)
+                continue
+            comparer.compare(reffile, l10n, mergepath, extra_tests)
+    return observers

new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/mozpath.py
@@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import posixpath
+import os
+import re
+
+'''
+Like os.path, with a reduced set of functions, and with normalized path
+separators (always use forward slashes).
+Also contains a few additional utilities not found in os.path.
+'''
+
+
+def normsep(path):
+    '''
+    Normalize path separators, by using forward slashes instead of whatever
+    os.sep is.
+    '''
+    if os.sep != '/':
+        path = path.replace(os.sep, '/')
+    if os.altsep and os.altsep != '/':
+        path = path.replace(os.altsep, '/')
+    return path
+
+
+def relpath(path, start):
+    rel = normsep(os.path.relpath(path, start))
+    return '' if rel == '.' else rel
+
+
+def realpath(path):
+    return normsep(os.path.realpath(path))
+
+
+def abspath(path):
+    return normsep(os.path.abspath(path))
+
+
+def join(*paths):
+    return normsep(os.path.join(*paths))
+
+
+def normpath(path):
+    return posixpath.normpath(normsep(path))
+
+
+def dirname(path):
+    return posixpath.dirname(normsep(path))
+
+
+def commonprefix(paths):
+    return posixpath.commonprefix([normsep(path) for path in paths])
+
+
+def basename(path):
+    return os.path.basename(path)
+
+
+def splitext(path):
+    return posixpath.splitext(normsep(path))
+
+
+def split(path):
+    '''
+    Return the normalized path as a list of its components.
+        split('foo/bar/baz') returns ['foo', 'bar', 'baz']
+    '''
+    return normsep(path).split('/')
+
+
+def basedir(path, bases):
+    '''
+    Given a list of directories (bases), return which one contains the given
+    path. If several matches are found, the deepest base directory is returned.
+        basedir('foo/bar/baz', ['foo', 'baz', 'foo/bar']) returns 'foo/bar'
+        ('foo' and 'foo/bar' both match, but 'foo/bar' is the deepest match)
+    '''
+    path = normsep(path)
+    bases = [normsep(b) for b in bases]
+    if path in bases:
+        return path
+    for b in sorted(bases, reverse=True):
+        if b == '' or path.startswith(b + '/'):
+            return b
+
+
+re_cache = {}
+
+
+def match(path, pattern):
+    '''
+    Return whether the given path matches the given pattern.
+    An asterisk can be used to match any string, including the null string, in
+    one part of the path:
+        'foo' matches '*', 'f*' or 'fo*o'
+    However, an asterisk matching a subdirectory may not match the null string:
+        'foo/bar' does *not* match 'foo/*/bar'
+    If the pattern matches one of the ancestor directories of the path, the
+    patch is considered matching:
+        'foo/bar' matches 'foo'
+    Two adjacent asterisks can be used to match files and zero or more
+    directories and subdirectories.
+        'foo/bar' matches 'foo/**/bar', or '**/bar'
+    '''
+    if not pattern:
+        return True
+    if pattern not in re_cache:
+        p = re.escape(pattern)
+        p = re.sub(r'(^|\\\/)\\\*\\\*\\\/', r'\1(?:.+/)?', p)
+        p = re.sub(r'(^|\\\/)\\\*\\\*$', r'(?:\1.+)?', p)
+        p = p.replace(r'\*', '[^/]*') + '(?:/.*)?$'
+        re_cache[pattern] = re.compile(p)
+    return re_cache[pattern].match(path) is not None
+
+
+def rebase(oldbase, base, relativepath):
+    '''
+    Return relativepath relative to base instead of oldbase.
+    '''
+    if base == oldbase:
+        return relativepath
+    if len(base) < len(oldbase):
+        assert basedir(oldbase, [base]) == base
+        relbase = relpath(oldbase, base)
+        result = join(relbase, relativepath)
+    else:
+        assert basedir(base, [oldbase]) == oldbase
+        relbase = relpath(base, oldbase)
+        result = relpath(relativepath, relbase)
+    result = normpath(result)
+    if relativepath.endswith('/') and not result.endswith('/'):
+        result += '/'
+    return result

--- a/third_party/python/compare-locales/compare_locales/parser.py
+++ b/third_party/python/compare-locales/compare_locales/parser.py
@@ -1,46 +1,62 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import bisect
 import codecs
+from collections import Counter
 import logging
 
+from fluent.syntax import FluentParser as FTLParser
+from fluent.syntax import ast as ftl
+
 __constructors = []
 
 
+# The allowed capabilities for the Parsers.  They define the exact strategy
+# used by ContentComparer.merge.
+
+# Don't perform any merging
+CAN_NONE = 0
+# Copy the entire reference file
+CAN_COPY = 1
+# Remove broken entities from localization
+CAN_SKIP = 2
+# Add missing and broken entities from the reference to localization
+CAN_MERGE = 4
+
+
 class EntityBase(object):
     '''
     Abstraction layer for a localizable entity.
     Currently supported are grammars of the form:
 
     1: pre white space
     2: entity definition
     3: entity key (name)
     4: entity value
     5: post white space
                                                  <--[1]
     <!ENTITY key "value">
 
     <-------[2]--------->
     '''
-    def __init__(self, ctx, pp, pre_comment,
+    def __init__(self, ctx, pre_comment,
                  span, pre_ws_span, def_span,
                  key_span, val_span, post_span):
         self.ctx = ctx
         self.span = span
         self.pre_ws_span = pre_ws_span
         self.def_span = def_span
         self.key_span = key_span
         self.val_span = val_span
         self.post_span = post_span
-        self.pp = pp
         self.pre_comment = pre_comment
         pass
 
     def position(self, offset=0):
         """Get the 1-based line and column of the character
         with given offset into the Entity.
 
         If offset is negative, return the end of the Entity.
@@ -72,52 +88,62 @@ class EntityBase(object):
         return self.ctx.contents[self.pre_ws_span[0]:self.pre_ws_span[1]]
 
     def get_def(self):
         return self.ctx.contents[self.def_span[0]:self.def_span[1]]
 
     def get_key(self):
         return self.ctx.contents[self.key_span[0]:self.key_span[1]]
 
-    def get_val(self):
-        return self.pp(self.ctx.contents[self.val_span[0]:self.val_span[1]])
-
     def get_raw_val(self):
         return self.ctx.contents[self.val_span[0]:self.val_span[1]]
 
     def get_post(self):
         return self.ctx.contents[self.post_span[0]:self.post_span[1]]
 
     # getters
 
     all = property(get_all)
     pre_ws = property(get_pre_ws)
     definition = property(get_def)
     key = property(get_key)
-    val = property(get_val)
+    val = property(get_raw_val)
     raw_val = property(get_raw_val)
     post = property(get_post)
 
     def __repr__(self):
         return self.key
 
+    re_br = re.compile('<br\s*/?>', re.U)
+    re_sgml = re.compile('</?\w+.*?>', re.U | re.M)
+
+    def count_words(self):
+        """Count the words in an English string.
+        Replace a couple of xml markup to make that safer, too.
+        """
+        value = self.re_br.sub(u'\n', self.val)
+        value = self.re_sgml.sub(u'', value)
+        return len(value.split())
+
+    def equals(self, other):
+        return self.key == other.key and self.val == other.val
+
 
 class Entity(EntityBase):
     pass
 
 
 class Comment(EntityBase):
     def __init__(self, ctx, span, pre_ws_span, def_span,
                  post_span):
         self.ctx = ctx
         self.span = span
         self.pre_ws_span = pre_ws_span
         self.def_span = def_span
         self.post_span = post_span
-        self.pp = lambda v: v
 
     @property
     def key(self):
         return None
 
     @property
     def val(self):
         return None
@@ -169,24 +195,23 @@ class Whitespace(EntityBase):
     '''Entity-like object representing an empty file with whitespace,
     if allowed
     '''
     def __init__(self, ctx, span):
         self.ctx = ctx
         self.key_span = self.val_span = self.span = span
         self.def_span = self.pre_ws_span = (span[0], span[0])
         self.post_span = (span[1], span[1])
-        self.pp = lambda v: v
 
     def __repr__(self):
         return self.raw_val
 
 
-class Parser:
-    canMerge = True
+class Parser(object):
+    capabilities = CAN_SKIP | CAN_MERGE
     tail = re.compile('\s+\Z')
 
     class Context(object):
         "Fixture for content and line numbers"
         def __init__(self, contents):
             self.contents = contents
             self._lines = None
 
@@ -228,33 +253,30 @@ class Parser:
     def parse(self):
         l = []
         m = {}
         for e in self:
             m[e.key] = len(l)
             l.append(e)
         return (l, m)
 
-    def postProcessValue(self, val):
-        return val
-
     def __iter__(self):
         return self.walk(onlyEntities=True)
 
     def walk(self, onlyEntities=False):
         if not self.ctx:
             # loading file failed, or we just didn't load anything
             return
         ctx = self.ctx
         contents = ctx.contents
         offset = 0
         entity, offset = self.getEntity(ctx, offset)
         while entity:
             if (not onlyEntities or
-                    type(entity) is Entity or
+                    isinstance(entity, Entity) or
                     type(entity) is Junk):
                 yield entity
             entity, offset = self.getEntity(ctx, offset)
         if len(contents) > offset:
             yield Junk(ctx, (offset, len(contents)))
 
     def getEntity(self, ctx, offset):
         m = self.reKey.match(ctx.contents, offset)
@@ -279,21 +301,28 @@ class Parser:
             if self.tail.match(ctx.contents, offset):
                 white_end = len(ctx.contents)
                 return (Whitespace(ctx, (offset, white_end)), white_end)
             else:
                 return (None, offset)
         return (Junk(ctx, (offset, junkend)), junkend)
 
     def createEntity(self, ctx, m):
-        pre_comment = unicode(self.last_comment) if self.last_comment else ''
-        self.last_comment = ''
-        return Entity(ctx, self.postProcessValue, pre_comment,
+        pre_comment = self.last_comment
+        self.last_comment = None
+        return Entity(ctx, pre_comment,
                       *[m.span(i) for i in xrange(6)])
 
+    @classmethod
+    def findDuplicates(cls, entities):
+        found = Counter(entity.key for entity in entities)
+        for entity_id, cnt in found.items():
+            if cnt > 1:
+                yield '{} occurs {} times'.format(entity_id, cnt)
+
 
 def getParser(path):
     for item in __constructors:
         if re.search(item[0], path):
             return item[1]
     raise UserWarning("Cannot find Parser")
 
 
@@ -306,16 +335,32 @@ def getParser(path):
 # 6: post comment (and white space) in the same line (dtd only)
 #                                            <--[1]
 # <!-- pre comments -->                      <--[2]
 # <!ENTITY key "value"> <!-- comment -->
 #
 # <-------[3]---------><------[6]------>
 
 
+class DTDEntity(Entity):
+    def value_position(self, offset=0):
+        # DTDChecker already returns tuples of (line, col) positions
+        if isinstance(offset, tuple):
+            line_pos, col_pos = offset
+            line, col = super(DTDEntity, self).value_position()
+            if line_pos == 1:
+                col = col + col_pos
+            else:
+                col = col_pos
+                line += line_pos - 1
+            return line, col
+        else:
+            return super(DTDEntity, self).value_position(offset)
+
+
 class DTDParser(Parser):
     # http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-NameStartChar
     # ":" | [A-Z] | "_" | [a-z] |
     # [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF]
     # | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] |
     # [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] |
     # [#x10000-#xEFFFF]
     CharMinusDash = u'\x09\x0A\x0D\u0020-\u002C\u002E-\uD7FF\uE000-\uFFFD'
@@ -352,38 +397,51 @@ class DTDParser(Parser):
         '''
         if offset is 0 and self.reHeader.match(ctx.contents):
             offset += 1
         entity, inneroffset = Parser.getEntity(self, ctx, offset)
         if (entity and isinstance(entity, Junk)) or entity is None:
             m = self.rePE.match(ctx.contents, offset)
             if m:
                 inneroffset = m.end()
-                self.last_comment = ''
-                entity = Entity(ctx, self.postProcessValue, '',
-                                *[m.span(i) for i in xrange(6)])
+                self.last_comment = None
+                entity = DTDEntity(ctx, '', *[m.span(i) for i in xrange(6)])
         return (entity, inneroffset)
 
     def createEntity(self, ctx, m):
         valspan = m.span('val')
         valspan = (valspan[0]+1, valspan[1]-1)
-        pre_comment = unicode(self.last_comment) if self.last_comment else ''
-        self.last_comment = ''
-        return Entity(ctx, self.postProcessValue, pre_comment,
-                      m.span(),
-                      m.span('pre'),
-                      m.span('entity'), m.span('key'), valspan,
-                      m.span('post'))
+        pre_comment = self.last_comment
+        self.last_comment = None
+        return DTDEntity(ctx, pre_comment,
+                         m.span(),
+                         m.span('pre'),
+                         m.span('entity'), m.span('key'), valspan,
+                         m.span('post'))
 
 
-class PropertiesParser(Parser):
+class PropertiesEntity(Entity):
     escape = re.compile(r'\\((?P<uni>u[0-9a-fA-F]{1,4})|'
                         '(?P<nl>\n\s*)|(?P<single>.))', re.M)
     known_escapes = {'n': '\n', 'r': '\r', 't': '\t', '\\': '\\'}
 
+    @property
+    def val(self):
+        def unescape(m):
+            found = m.groupdict()
+            if found['uni']:
+                return unichr(int(found['uni'][1:], 16))
+            if found['nl']:
+                return ''
+            return self.known_escapes.get(found['single'], found['single'])
+
+        return self.escape.sub(unescape, self.raw_val)
+
+
+class PropertiesParser(Parser):
     def __init__(self):
         self.reKey = re.compile('^(\s*)'
                                 '([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', re.M)
         self.reComment = re.compile('(\s*)(((?:[#!][^\n]*\n?)+))', re.M)
         self._escapedEnd = re.compile(r'\\+$')
         self._trailingWS = re.compile(r'\s*(?:\n|\Z)', re.M)
         Parser.__init__(self)
 
@@ -419,61 +477,48 @@ class PropertiesParser(Parser):
                 if len(_e.group()) % 2 == 0:
                     break
                 startline = offset
             # strip trailing whitespace
             ws = self._trailingWS.search(contents, startline)
             if ws:
                 endval = ws.start()
                 offset = ws.end()
-            pre_comment = (unicode(self.last_comment) if self.last_comment
-                           else '')
-            self.last_comment = ''
-            entity = Entity(ctx, self.postProcessValue, pre_comment,
-                            (m.start(), offset),   # full span
-                            m.span(1),  # leading whitespan
-                            (m.start(2), offset),   # entity def span
-                            m.span(2),   # key span
-                            (m.end(), endval),   # value span
-                            (offset, offset))  # post comment span, empty
+            pre_comment = self.last_comment
+            self.last_comment = None
+            entity = PropertiesEntity(
+                ctx, pre_comment,
+                (m.start(), offset),   # full span
+                m.span(1),  # leading whitespan
+                (m.start(2), offset),   # entity def span
+                m.span(2),   # key span
+                (m.end(), endval),   # value span
+                (offset, offset))  # post comment span, empty
             return (entity, offset)
         return self.getTrailing(ctx, offset, self.reKey, self.reComment)
 
-    def postProcessValue(self, val):
-
-        def unescape(m):
-            found = m.groupdict()
-            if found['uni']:
-                return unichr(int(found['uni'][1:], 16))
-            if found['nl']:
-                return ''
-            return self.known_escapes.get(found['single'], found['single'])
-        val = self.escape.sub(unescape, val)
-        return val
-
 
 class DefinesInstruction(EntityBase):
     '''Entity-like object representing processing instructions in inc files
     '''
     def __init__(self, ctx, span, pre_ws_span, def_span, val_span, post_span):
         self.ctx = ctx
         self.span = span
         self.pre_ws_span = pre_ws_span
         self.def_span = def_span
         self.key_span = self.val_span = val_span
         self.post_span = post_span
-        self.pp = lambda v: v
 
     def __repr__(self):
         return self.raw_val
 
 
 class DefinesParser(Parser):
     # can't merge, #unfilter needs to be the last item, which we don't support
-    canMerge = False
+    capabilities = CAN_COPY
     tail = re.compile(r'(?!)')  # never match
 
     def __init__(self):
         self.reComment = re.compile(
             '((?:[ \t]*\n)*)'
             '((?:^# .*?(?:\n|\Z))+)'
             '((?:[ \t]*(?:\n|\Z))*)', re.M)
         self.reKey = re.compile('((?:[ \t]*\n)*)'
@@ -511,17 +556,16 @@ class IniSection(EntityBase):
     '''
     def __init__(self, ctx, span, pre_ws_span, def_span, val_span, post_span):
         self.ctx = ctx
         self.span = span
         self.pre_ws_span = pre_ws_span
         self.def_span = def_span
         self.key_span = self.val_span = val_span
         self.post_span = post_span
-        self.pp = lambda v: v
 
     def __repr__(self):
         return self.raw_val
 
 
 class IniParser(Parser):
     '''
     Parse files of the form:
@@ -561,12 +605,113 @@ class IniParser(Parser):
         m = self.reKey.match(contents, offset)
         if m:
             offset = m.end()
             return (self.createEntity(ctx, m), offset)
         return self.getTrailing(ctx, offset,
                                 self.reComment, self.reSection, self.reKey)
 
 
+class FluentAttribute(EntityBase):
+    ignored_fields = ['span']
+
+    def __init__(self, entity, attr_node):
+        self.ctx = entity.ctx
+        self.attr = attr_node
+        self.key_span = (attr_node.id.span.start, attr_node.id.span.end)
+        self.val_span = (attr_node.value.span.start, attr_node.value.span.end)
+
+    def equals(self, other):
+        if not isinstance(other, FluentAttribute):
+            return False
+        return self.attr.equals(
+            other.attr, ignored_fields=self.ignored_fields)
+
+
+class FluentEntity(Entity):
+    # Fields ignored when comparing two entities.
+    ignored_fields = ['comment', 'span', 'tags']
+
+    def __init__(self, ctx, entry):
+        start = entry.span.start
+        end = entry.span.end
+
+        self.ctx = ctx
+        self.span = (start, end)
+
+        self.key_span = (entry.id.span.start, entry.id.span.end)
+
+        if entry.value is not None:
+            self.val_span = (entry.value.span.start, entry.value.span.end)
+        else:
+            self.val_span = (0, 0)
+
+        self.entry = entry
+
+    _word_count = None
+
+    def count_words(self):
+        if self._word_count is None:
+            self._word_count = 0
+
+            def count_words(node):
+                if isinstance(node, ftl.TextElement):
+                    self._word_count += len(node.value.split())
+                return node
+
+            self.entry.traverse(count_words)
+
+        return self._word_count
+
+    def equals(self, other):
+        return self.entry.equals(
+            other.entry, ignored_fields=self.ignored_fields)
+
+    # Positions yielded by FluentChecker.check are absolute offsets from the
+    # beginning of the file.  This is different from the base Checker behavior
+    # which yields offsets from the beginning of the current entity's value.
+    def position(self, pos=None):
+        if pos is None:
+            pos = self.entry.span.start
+        return self.ctx.lines(pos)[0]
+
+    # FluentEntities don't differentiate between entity and value positions
+    # because all positions are absolute from the beginning of the file.
+    def value_position(self, pos=None):
+        return self.position(pos)
+
+    @property
+    def attributes(self):
+        for attr_node in self.entry.attributes:
+            yield FluentAttribute(self, attr_node)
+
+
+class FluentParser(Parser):
+    capabilities = CAN_SKIP
+
+    def __init__(self):
+        super(FluentParser, self).__init__()
+        self.ftl_parser = FTLParser()
+
+    def walk(self, onlyEntities=False):
+        if not self.ctx:
+            # loading file failed, or we just didn't load anything
+            return
+        resource = self.ftl_parser.parse(self.ctx.contents)
+        for entry in resource.body:
+            if isinstance(entry, ftl.Message):
+                yield FluentEntity(self.ctx, entry)
+            elif isinstance(entry, ftl.Junk):
+                start = entry.span.start
+                end = entry.span.end
+                # strip leading whitespace
+                start += re.match('\s*', entry.content).end()
+                # strip trailing whitespace
+                ws, we = re.search('\s*$', entry.content).span()
+                end -= we - ws
+                yield Junk(self.ctx, (start, end))
+
+
 __constructors = [('\\.dtd$', DTDParser()),
                   ('\\.properties$', PropertiesParser()),
                   ('\\.ini$', IniParser()),
-                  ('\\.inc$', DefinesParser())]
+                  ('\\.inc$', DefinesParser()),
+                  ('\\.ftl$', FluentParser())]

--- a/third_party/python/compare-locales/compare_locales/paths.py
+++ b/third_party/python/compare-locales/compare_locales/paths.py
@@ -1,69 +1,566 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import os.path
 import os
+import re
 from ConfigParser import ConfigParser, NoSectionError, NoOptionError
-from urlparse import urlparse, urljoin
-from urllib import pathname2url, url2pathname
-from urllib2 import urlopen
 from collections import defaultdict
-from compare_locales import util
+import errno
+import itertools
+import logging
+from compare_locales import util, mozpath
+import pytoml as toml
+
+
+class Matcher(object):
+    '''Path pattern matcher
+    Supports path matching similar to mozpath.match(), but does
+    not match trailing file paths without trailing wildcards.
+    Also gets a prefix, which is the path before the first wildcard,
+    which is good for filesystem iterations, and allows to replace
+    the own matches in a path on a different Matcher. compare-locales
+    uses that to transform l10n and en-US paths back and forth.
+    '''
+
+    def __init__(self, pattern):
+        '''Create regular expression similar to mozpath.match().
+        '''
+        prefix = pattern.split("*", 1)[0]
+        p = re.escape(pattern)
+        p = re.sub(r'(^|\\\/)\\\*\\\*\\\/', r'\1(.+/)?', p)
+        p = re.sub(r'(^|\\\/)\\\*\\\*$', r'(\1.+)?', p)
+        p = p.replace(r'\*', '([^/]*)') + '$'
+        r = re.escape(pattern)
+        r = re.sub(r'(^|\\\/)\\\*\\\*\\\/', r'\\\\0', r)
+        r = re.sub(r'(^|\\\/)\\\*\\\*$', r'\\\\0', r)
+        r = r.replace(r'\*', r'\\0')
+        backref = itertools.count(1)
+        r = re.sub(r'\\0', lambda m: '\\%s' % backref.next(), r)
+        r = re.sub(r'\\(.)', r'\1', r)
+        self.prefix = prefix
+        self.regex = re.compile(p)
+        self.placable = r
+
+    def match(self, path):
+        '''
+        True if the given path matches the file pattern.
+        '''
+        return self.regex.match(path) is not None
+
+    def sub(self, other, path):
+        '''
+        Replace the wildcard matches in this pattern into the
+        pattern of the other Match object.
+        '''
+        if not self.match(path):
+            return None
+        return self.regex.sub(other.placable, path)
+
+
+class ProjectConfig(object):
+    '''Abstraction of l10n project configuration data.
+    '''
+
+    def __init__(self):
+        self.filter_py = None  # legacy filter code
+        # {
+        #  'l10n': pattern,
+        #  'reference': pattern,  # optional
+        #  'locales': [],  # optional
+        #  'test': [],  # optional
+        # }
+        self.paths = []
+        self.rules = []
+        self.locales = []
+        self.environ = {}
+        self.children = []
+        self._cache = None
+
+    variable = re.compile('{\s*([\w]+)\s*}')
+
+    def expand(self, path, env=None):
+        if env is None:
+            env = {}
+
+        def _expand(m):
+            _var = m.group(1)
+            for _env in (env, self.environ):
+                if _var in _env:
+                    return self.expand(_env[_var], env)
+            return '{{{}}}'.format(_var)
+        return self.variable.sub(_expand, path)
+
+    def lazy_expand(self, pattern):
+        def lazy_l10n_expanded_pattern(env):
+            return Matcher(self.expand(pattern, env))
+        return lazy_l10n_expanded_pattern
+
+    def add_global_environment(self, **kwargs):
+        self.add_environment(**kwargs)
+        for child in self.children:
+            child.add_global_environment(**kwargs)
+
+    def add_environment(self, **kwargs):
+        self.environ.update(kwargs)
+
+    def add_paths(self, *paths):
+        '''Add path dictionaries to this config.
+        The dictionaries must have a `l10n` key. For monolingual files,
+        `reference` is also required.
+        An optional key `test` is allowed to enable additional tests for this
+        path pattern.
+        '''
+
+        for d in paths:
+            rv = {
+                'l10n': self.lazy_expand(d['l10n']),
+                'module': d.get('module')
+            }
+            if 'reference' in d:
+                rv['reference'] = Matcher(d['reference'])
+            if 'test' in d:
+                rv['test'] = d['test']
+            if 'locales' in d:
+                rv['locales'] = d['locales'][:]
+            self.paths.append(rv)
+
+    def set_filter_py(self, filter):
+        '''Set legacy filter.py code.
+        Assert that no rules are set.
+        Also, normalize output already here.
+        '''
+        assert not self.rules
+
+        def filter_(module, path, entity=None):
+            try:
+                rv = filter(module, path, entity=entity)
+            except:
+                return 'error'
+            rv = {
+                True: 'error',
+                False: 'ignore',
+                'report': 'warning'
+            }.get(rv, rv)
+            assert rv in ('error', 'ignore', 'warning', None)
+            return rv
+        self.filter_py = filter_
+
+    def add_rules(self, *rules):
+        '''Add rules to filter on.
+        Assert that there's no legacy filter.py code hooked up.
+        '''
+        assert self.filter_py is None
+        for rule in rules:
+            self.rules.extend(self._compile_rule(rule))
+
+    def add_child(self, child):
+        self.children.append(child)
+
+    def set_locales(self, locales, deep=False):
+        self.locales = locales
+        for child in self.children:
+            if not child.locales or deep:
+                child.set_locales(locales, deep=True)
+            else:
+                locs = [loc for loc in locales if loc in child.locales]
+                child.set_locales(locs)
+
+    @property
+    def configs(self):
+        'Recursively get all configs in this project and its children'
+        yield self
+        for child in self.children:
+            for config in child.configs:
+                yield config
+
+    def filter(self, l10n_file, entity=None):
+        '''Filter a localization file or entities within, according to
+        this configuration file.'''
+        if self.filter_py is not None:
+            return self.filter_py(l10n_file.module, l10n_file.file,
+                                  entity=entity)
+        rv = self._filter(l10n_file, entity=entity)
+        if rv is None:
+            return 'ignore'
+        return rv
+
+    class FilterCache(object):
+        def __init__(self, locale):
+            self.locale = locale
+            self.rules = []
+            self.l10n_paths = []
+
+    def cache(self, locale):
+        if self._cache and self._cache.locale == locale:
+            return self._cache
+        self._cache = self.FilterCache(locale)
+        for paths in self.paths:
+            self._cache.l10n_paths.append(paths['l10n']({
+                "locale": locale
+            }))
+        for rule in self.rules:
+            cached_rule = rule.copy()
+            cached_rule['path'] = rule['path']({
+                "locale": locale
+            })
+            self._cache.rules.append(cached_rule)
+        return self._cache
+
+    def _filter(self, l10n_file, entity=None):
+        actions = set(
+            child._filter(l10n_file, entity=entity)
+            for child in self.children)
+        if 'error' in actions:
+            # return early if we know we'll error
+            return 'error'
+
+        cached = self.cache(l10n_file.locale)
+        if any(p.match(l10n_file.fullpath) for p in cached.l10n_paths):
+            action = 'error'
+            for rule in reversed(cached.rules):
+                if not rule['path'].match(l10n_file.fullpath):
+                    continue
+                if ('key' in rule) ^ (entity is not None):
+                    # key/file mismatch, not a matching rule
+                    continue
+                if 'key' in rule and not rule['key'].match(entity):
+                    continue
+                action = rule['action']
+                break
+            actions.add(action)
+        if 'error' in actions:
+            return 'error'
+        if 'warning' in actions:
+            return 'warning'
+        if 'ignore' in actions:
+            return 'ignore'
+
+    def _compile_rule(self, rule):
+        assert 'path' in rule
+        if isinstance(rule['path'], list):
+            for path in rule['path']:
+                _rule = rule.copy()
+                _rule['path'] = self.lazy_expand(path)
+                for __rule in self._compile_rule(_rule):
+                    yield __rule
+            return
+        if isinstance(rule['path'], basestring):
+            rule['path'] = self.lazy_expand(rule['path'])
+        if 'key' not in rule:
+            yield rule
+            return
+        if not isinstance(rule['key'], basestring):
+            for key in rule['key']:
+                _rule = rule.copy()
+                _rule['key'] = key
+                for __rule in self._compile_rule(_rule):
+                    yield __rule
+            return
+        rule = rule.copy()
+        key = rule['key']
+        if key.startswith('re:'):
+            key = key[3:]
+        else:
+            key = re.escape(key) + '$'
+        rule['key'] = re.compile(key)
+        yield rule
+
+
+class ProjectFiles(object):
+    '''Iterable object to get all files and tests for a locale and a
+    list of ProjectConfigs.
+    '''
+    def __init__(self, locale, projects, mergebase=None):
+        self.locale = locale
+        self.matchers = []
+        self.mergebase = mergebase
+        configs = []
+        for project in projects:
+            configs.extend(project.configs)
+        for pc in configs:
+            if locale not in pc.locales:
+                continue
+            for paths in pc.paths:
+                if 'locales' in paths and locale not in paths['locales']:
+                    continue
+                m = {
+                    'l10n': paths['l10n']({
+                        "locale": locale
+                    }),
+                    'module': paths.get('module'),
+                }
+                if 'reference' in paths:
+                    m['reference'] = paths['reference']
+                if self.mergebase is not None:
+                    m['merge'] = paths['l10n']({
+                        "locale": locale,
+                        "l10n_base": self.mergebase
+                    })
+                m['test'] = set(paths.get('test', []))
+                if 'locales' in paths:
+                    m['locales'] = paths['locales'][:]
+                self.matchers.append(m)
+        self.matchers.reverse()  # we always iterate last first
+        # Remove duplicate patterns, comparing each matcher
+        # against all other matchers.
+        # Avoid n^2 comparisons by only scanning the upper triangle
+        # of a n x n matrix of all possible combinations.
+        # Using enumerate and keeping track of indexes, as we can't
+        # modify the list while iterating over it.
+        drops = set()  # duplicate matchers to remove
+        for i, m in enumerate(self.matchers[:-1]):
+            if i in drops:
+                continue  # we're dropping this anyway, don't search again
+            for i_, m_ in enumerate(self.matchers[(i+1):]):
+                if (mozpath.realpath(m['l10n'].prefix) !=
+                        mozpath.realpath(m_['l10n'].prefix)):
+                    # ok, not the same thing, continue
+                    continue
+                # check that we're comparing the same thing
+                if 'reference' in m:
+                    if (mozpath.realpath(m['reference'].prefix) !=
+                            mozpath.realpath(m_.get('reference').prefix)):
+                        raise RuntimeError('Mismatch in reference for ' +
+                                           mozpath.realpath(m['l10n'].prefix))
+                drops.add(i_ + i + 1)
+                m['test'] |= m_['test']
+        drops = sorted(drops, reverse=True)
+        for i in drops:
+            del self.matchers[i]
+
+    def __iter__(self):
+        known = {}
+        for matchers in self.matchers:
+            matcher = matchers['l10n']
+            for path in self._files(matcher):
+                if path not in known:
+                    known[path] = {'test': matchers.get('test')}
+                    if 'reference' in matchers:
+                        known[path]['reference'] = matcher.sub(
+                            matchers['reference'], path)
+                    if 'merge' in matchers:
+                        known[path]['merge'] = matcher.sub(
+                            matchers['merge'], path)
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            for path in self._files(matcher):
+                l10npath = matcher.sub(matchers['l10n'], path)
+                if l10npath not in known:
+                    known[l10npath] = {
+                        'reference': path,
+                        'test': matchers.get('test')
+                    }
+                    if 'merge' in matchers:
+                        known[l10npath]['merge'] = \
+                            matcher.sub(matchers['merge'], path)
+        for path, d in sorted(known.items()):
+            yield (path, d.get('reference'), d.get('merge'), d['test'])
+
+    def _files(self, matcher):
+        '''Base implementation of getting all files in a hierarchy
+        using the file system.
+        Subclasses might replace this method to support different IO
+        patterns.
+        '''
+        base = matcher.prefix
+        if os.path.isfile(base):
+            if matcher.match(base):
+                yield base
+            return
+        for d, dirs, files in os.walk(base):
+            for f in files:
+                p = mozpath.join(d, f)
+                if matcher.match(p):
+                    yield p
+
+    def match(self, path):
+        '''Return the tuple of l10n_path, reference, mergepath, tests
+        if the given path matches any config, otherwise None.
+
+        This routine doesn't check that the files actually exist.
+        '''
+        for matchers in self.matchers:
+            matcher = matchers['l10n']
+            if matcher.match(path):
+                ref = merge = None
+                if 'reference' in matchers:
+                    ref = matcher.sub(matchers['reference'], path)
+                if 'merge' in matchers:
+                    merge = matcher.sub(matchers['merge'], path)
+                return path, ref, merge, matchers.get('test')
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            if matcher.match(path):
+                merge = None
+                l10n = matcher.sub(matchers['l10n'], path)
+                if 'merge' in matchers:
+                    merge = matcher.sub(matchers['merge'], path)
+                return l10n, path, merge, matchers.get('test')
+
+
+class ConfigNotFound(EnvironmentError):
+    def __init__(self, path):
+        super(ConfigNotFound, self).__init__(
+            errno.ENOENT,
+            'Configuration file not found',
+            path)
+
+
+class TOMLParser(object):
+    @classmethod
+    def parse(cls, path, env=None, ignore_missing_includes=False):
+        parser = cls(path, env=env,
+                     ignore_missing_includes=ignore_missing_includes)
+        parser.load()
+        parser.processEnv()
+        parser.processPaths()
+        parser.processFilters()
+        parser.processIncludes()
+        parser.processLocales()
+        return parser.asConfig()
+
+    def __init__(self, path, env=None, ignore_missing_includes=False):
+        self.path = path
+        self.env = env if env is not None else {}
+        self.ignore_missing_includes = ignore_missing_includes
+        self.data = None
+        self.pc = ProjectConfig()
+        self.pc.PATH = path
+
+    def load(self):
+        try:
+            with open(self.path, 'rb') as fin:
+                self.data = toml.load(fin)
+        except:
+            raise ConfigNotFound(self.path)
+
+    def processEnv(self):
+        assert self.data is not None
+        self.pc.add_environment(**self.data.get('env', {}))
+
+    def processLocales(self):
+        assert self.data is not None
+        if 'locales' in self.data:
+            self.pc.set_locales(self.data['locales'])
+
+    def processPaths(self):
+        assert self.data is not None
+        for data in self.data.get('paths', []):
+            l10n = data['l10n']
+            if not l10n.startswith('{'):
+                # l10n isn't relative to a variable, expand
+                l10n = self.resolvepath(l10n)
+            paths = {
+                "l10n": l10n,
+            }
+            if 'locales' in data:
+                paths['locales'] = data['locales']
+            if 'reference' in data:
+                paths['reference'] = self.resolvepath(data['reference'])
+            self.pc.add_paths(paths)
+
+    def processFilters(self):
+        assert self.data is not None
+        for data in self.data.get('filters', []):
+            paths = data['path']
+            if isinstance(paths, basestring):
+                paths = [paths]
+            # expand if path isn't relative to a variable
+            paths = [
+                self.resolvepath(path) if not path.startswith('{')
+                else path
+                for path in paths
+            ]
+            rule = {
+                "path": paths,
+                "action": data['action']
+            }
+            if 'key' in data:
+                rule['key'] = data['key']
+            self.pc.add_rules(rule)
+
+    def processIncludes(self):
+        assert self.data is not None
+        if 'includes' not in self.data:
+            return
+        for include in self.data['includes']:
+            p = include['path']
+            p = self.resolvepath(p)
+            try:
+                child = self.parse(
+                    p, env=self.env,
+                    ignore_missing_includes=self.ignore_missing_includes
+                )
+            except ConfigNotFound as e:
+                if not self.ignore_missing_includes:
+                    raise
+                (logging
+                    .getLogger('compare-locales.io')
+                    .error('%s: %s', e.strerror, e.filename))
+                continue
+            self.pc.add_child(child)
+
+    def resolvepath(self, path):
+        path = self.pc.expand(path, env=self.env)
+        path = mozpath.join(
+            mozpath.dirname(self.path),
+            self.data.get('basepath', '.'),
+            path)
+        return mozpath.normpath(path)
+
+    def asConfig(self):
+        return self.pc
 
 
 class L10nConfigParser(object):
     '''Helper class to gather application information from ini files.
 
     This class is working on synchronous open to read files or web data.
     Subclass this and overwrite loadConfigs and addChild if you need async.
     '''
     def __init__(self, inipath, **kwargs):
         """Constructor for L10nConfigParsers
 
         inipath -- l10n.ini path
         Optional keyword arguments are fowarded to the inner ConfigParser as
         defaults.
         """
-        if os.path.isabs(inipath):
-            self.inipath = 'file:%s' % pathname2url(inipath)
-        else:
-            pwdurl = 'file:%s/' % pathname2url(os.getcwd())
-            self.inipath = urljoin(pwdurl, inipath)
+        self.inipath = mozpath.normpath(inipath)
         # l10n.ini files can import other l10n.ini files, store the
         # corresponding L10nConfigParsers
         self.children = []
         # we really only care about the l10n directories described in l10n.ini
         self.dirs = []
         # optional defaults to be passed to the inner ConfigParser (unused?)
         self.defaults = kwargs
 
     def getDepth(self, cp):
         '''Get the depth for the comparison from the parsed l10n.ini.
-
-        Overloadable to get the source depth for fennec and friends.
         '''
         try:
             depth = cp.get('general', 'depth')
         except:
             depth = '.'
         return depth
 
     def getFilters(self):
         '''Get the test functions from this ConfigParser and all children.
 
         Only works with synchronous loads, used by compare-locales, which
         is local anyway.
         '''
-        filterurl = urljoin(self.inipath, 'filter.py')
+        filter_path = mozpath.join(mozpath.dirname(self.inipath), 'filter.py')
         try:
             l = {}
-            execfile(url2pathname(urlparse(filterurl).path), {}, l)
+            execfile(filter_path, {}, l)
             if 'test' in l and callable(l['test']):
                 filters = [l['test']]
             else:
                 filters = []
         except:
             filters = []
 
         for c in self.children:
@@ -73,334 +570,197 @@ class L10nConfigParser(object):
 
     def loadConfigs(self):
         """Entry point to load the l10n.ini file this Parser refers to.
 
         This implementation uses synchronous loads, subclasses might overload
         this behaviour. If you do, make sure to pass a file-like object
         to onLoadConfig.
         """
-        self.onLoadConfig(urlopen(self.inipath))
-
-    def onLoadConfig(self, inifile):
-        """Parse a file-like object for the loaded l10n.ini file."""
         cp = ConfigParser(self.defaults)
-        cp.readfp(inifile)
+        cp.read(self.inipath)
         depth = self.getDepth(cp)
-        self.baseurl = urljoin(self.inipath, depth)
+        self.base = mozpath.join(mozpath.dirname(self.inipath), depth)
         # create child loaders for any other l10n.ini files to be included
         try:
             for title, path in cp.items('includes'):
                 # skip default items
                 if title in self.defaults:
                     continue
                 # add child config parser
                 self.addChild(title, path, cp)
         except NoSectionError:
             pass
         # try to load the "dirs" defined in the "compare" section
         try:
             self.dirs.extend(cp.get('compare', 'dirs').split())
         except (NoOptionError, NoSectionError):
             pass
-        # try getting a top level compare dir, as used for fennec
-        try:
-            self.tld = cp.get('compare', 'tld')
-            # remove tld from comparison dirs
-            if self.tld in self.dirs:
-                self.dirs.remove(self.tld)
-        except (NoOptionError, NoSectionError):
-            self.tld = None
         # try to set "all_path" and "all_url"
         try:
-            self.all_path = cp.get('general', 'all')
-            self.all_url = urljoin(self.baseurl, self.all_path)
+            self.all_path = mozpath.join(self.base, cp.get('general', 'all'))
         except (NoOptionError, NoSectionError):
             self.all_path = None
-            self.all_url = None
         return cp
 
     def addChild(self, title, path, orig_cp):
         """Create a child L10nConfigParser and load it.
 
         title -- indicates the module's name
         path -- indicates the path to the module's l10n.ini file
         orig_cp -- the configuration parser of this l10n.ini
         """
-        cp = L10nConfigParser(urljoin(self.baseurl, path), **self.defaults)
+        cp = L10nConfigParser(mozpath.join(self.base, path), **self.defaults)
         cp.loadConfigs()
         self.children.append(cp)
 
-    def getTLDPathsTuple(self, basepath):
-        """Given the basepath, return the path fragments to be used for
-        self.tld. For build runs, this is (basepath, self.tld), for
-        source runs, just (basepath,).
-
-        @see overwritten method in SourceTreeConfigParser.
-        """
-        return (basepath, self.tld)
-
     def dirsIter(self):
         """Iterate over all dirs and our base path for this l10n.ini"""
-        url = urlparse(self.baseurl)
-        basepath = url2pathname(url.path)
-        if self.tld is not None:
-            yield self.tld, self.getTLDPathsTuple(basepath)
         for dir in self.dirs:
-            yield dir, (basepath, dir)
+            yield dir, (self.base, dir)
 
     def directories(self):
         """Iterate over all dirs and base paths for this l10n.ini as well
         as the included ones.
         """
         for t in self.dirsIter():
             yield t
         for child in self.children:
             for t in child.directories():
                 yield t
 
     def allLocales(self):
         """Return a list of all the locales of this project"""
-        return util.parseLocales(urlopen(self.all_url).read())
+        return util.parseLocales(open(self.all_path).read())
 
 
 class SourceTreeConfigParser(L10nConfigParser):
     '''Subclassing L10nConfigParser to work with just the repos
     checked out next to each other instead of intermingled like
     we do for real builds.
     '''
 
-    def __init__(self, inipath, basepath, redirects):
+    def __init__(self, inipath, base, redirects):
         '''Add additional arguments basepath.
 
         basepath is used to resolve local paths via branchnames.
         redirects is used in unified repository, mapping upstream
         repos to local clones.
         '''
         L10nConfigParser.__init__(self, inipath)
-        self.basepath = basepath
+        self.base = base
         self.redirects = redirects
-        self.tld = None
-
-    def getDepth(self, cp):
-        '''Get the depth for the comparison from the parsed l10n.ini.
-
-        Overloaded to get the source depth for fennec and friends.
-        '''
-        try:
-            depth = cp.get('general', 'source-depth')
-        except:
-            try:
-                depth = cp.get('general', 'depth')
-            except:
-                depth = '.'
-        return depth
 
     def addChild(self, title, path, orig_cp):
         # check if there's a section with details for this include
         # we might have to check a different repo, or even VCS
         # for example, projects like "mail" indicate in
         # an "include_" section where to find the l10n.ini for "toolkit"
         details = 'include_' + title
         if orig_cp.has_section(details):
             branch = orig_cp.get(details, 'mozilla')
             branch = self.redirects.get(branch, branch)
             inipath = orig_cp.get(details, 'l10n.ini')
-            path = self.basepath + '/' + branch + '/' + inipath
+            path = mozpath.join(self.base, branch, inipath)
         else:
-            path = urljoin(self.baseurl, path)
-        cp = SourceTreeConfigParser(path, self.basepath, self.redirects,
+            path = mozpath.join(self.base, path)
+        cp = SourceTreeConfigParser(path, self.base, self.redirects,
                                     **self.defaults)
         cp.loadConfigs()
         self.children.append(cp)
 
-    def getTLDPathsTuple(self, basepath):
-        """Overwrite L10nConfigParser's getTLDPathsTuple to just return
-        the basepath.
-        """
-        return (basepath, )
-
 
 class File(object):
 
     def __init__(self, fullpath, file, module=None, locale=None):
         self.fullpath = fullpath
         self.file = file
         self.module = module
         self.locale = locale
         pass
 
     def getContents(self):
         # open with universal line ending support and read
         return open(self.fullpath, 'rU').read()
 
-    def __hash__(self):
+    @property
+    def localpath(self):
         f = self.file
         if self.module:
-            f = self.module + '/' + f
-        return hash(f)
+            f = mozpath.join(self.module, f)
+        return f
+
+    def __hash__(self):
+        return hash(self.localpath)
 
     def __str__(self):
         return self.fullpath
 
     def __cmp__(self, other):
         if not isinstance(other, File):
             raise NotImplementedError
         rv = cmp(self.module, other.module)
         if rv != 0:
             return rv
         return cmp(self.file, other.file)
 
 
-class EnumerateDir(object):
-    ignore_dirs = ['CVS', '.svn', '.hg', '.git']
-
-    def __init__(self, basepath, module='', locale=None, ignore_subdirs=[]):
-        self.basepath = basepath
-        self.module = module
-        self.locale = locale
-        self.ignore_subdirs = ignore_subdirs
-        pass
-
-    def cloneFile(self, other):
-        '''
-        Return a File object that this enumerator would return, if it had it.
-        '''
-        return File(os.path.join(self.basepath, other.file), other.file,
-                    self.module, self.locale)
-
-    def __iter__(self):
-        # our local dirs are given as a tuple of path segments, starting off
-        # with an empty sequence for the basepath.
-        dirs = [()]
-        while dirs:
-            dir = dirs.pop(0)
-            fulldir = os.path.join(self.basepath, *dir)
-            try:
-                entries = os.listdir(fulldir)
-            except OSError:
-                # we probably just started off in a non-existing dir, ignore
-                continue
-            entries.sort()
-            for entry in entries:
-                leaf = os.path.join(fulldir, entry)
-                if os.path.isdir(leaf):
-                    if entry not in self.ignore_dirs and \
-                        leaf not in [os.path.join(self.basepath, d)
-                                     for d in self.ignore_subdirs]:
-                        dirs.append(dir + (entry,))
-                    continue
-                yield File(leaf, '/'.join(dir + (entry,)),
-                           self.module, self.locale)
-
-
-class LocalesWrap(object):
-
-    def __init__(self, base, module, locales, ignore_subdirs=[]):
-        self.base = base
-        self.module = module
-        self.locales = locales
-        self.ignore_subdirs = ignore_subdirs
-
-    def __iter__(self):
-        for locale in self.locales:
-            path = os.path.join(self.base, locale, self.module)
-            yield (locale, EnumerateDir(path, self.module, locale,
-                                        self.ignore_subdirs))
-
-
 class EnumerateApp(object):
     reference = 'en-US'
 
     def __init__(self, inipath, l10nbase, locales=None):
         self.setupConfigParser(inipath)
         self.modules = defaultdict(dict)
-        self.l10nbase = os.path.abspath(l10nbase)
+        self.l10nbase = mozpath.abspath(l10nbase)
         self.filters = []
-        drive, tail = os.path.splitdrive(inipath)
         self.addFilters(*self.config.getFilters())
         self.locales = locales or self.config.allLocales()
         self.locales.sort()
 
     def setupConfigParser(self, inipath):
         self.config = L10nConfigParser(inipath)
         self.config.loadConfigs()
 
     def addFilters(self, *args):
         self.filters += args
 
-    value_map = {None: None, 'error': 0, 'ignore': 1, 'report': 2}
+    def asConfig(self):
+        config = ProjectConfig()
+        self._config_for_ini(config, self.config)
+        filters = self.config.getFilters()
+        if filters:
+            config.set_filter_py(filters[0])
+        config.locales += self.locales
+        return config
 
-    def filter(self, l10n_file, entity=None):
-        '''Go through all added filters, and,
-        - map "error" -> 0, "ignore" -> 1, "report" -> 2
-        - if filter.test returns a bool, map that to
-            False -> "ignore" (1), True -> "error" (0)
-        - take the max of all reported
-        '''
-        rv = 0
-        for f in reversed(self.filters):
-            try:
-                _r = f(l10n_file.module, l10n_file.file, entity)
-            except:
-                # XXX error handling
-                continue
-            if isinstance(_r, bool):
-                _r = [1, 0][_r]
-            else:
-                # map string return value to int, default to 'error',
-                # None is None
-                _r = self.value_map.get(_r, 0)
-            if _r is not None:
-                rv = max(rv, _r)
-        return ['error', 'ignore', 'report'][rv]
-
-    def __iter__(self):
-        '''
-        Iterate over all modules, return en-US directory enumerator, and an
-        iterator over all locales in each iteration. Per locale, the locale
-        code and an directory enumerator will be given.
-        '''
-        dirmap = dict(self.config.directories())
-        mods = dirmap.keys()
-        mods.sort()
-        for mod in mods:
-            if self.reference == 'en-US':
-                base = os.path.join(*(dirmap[mod] + ('locales', 'en-US')))
-            else:
-                base = os.path.join(self.l10nbase, self.reference, mod)
-            yield (mod, EnumerateDir(base, mod, self.reference),
-                   LocalesWrap(self.l10nbase, mod, self.locales,
-                   [m[len(mod)+1:] for m in mods if m.startswith(mod+'/')]))
+    def _config_for_ini(self, projectconfig, aConfig):
+        for k, (basepath, module) in aConfig.dirsIter():
+            paths = {
+                'module': module,
+                'reference': mozpath.normpath('%s/%s/locales/en-US/**' %
+                                              (basepath, module)),
+                'l10n': mozpath.normpath('{l10n_base}/{locale}/%s/**' %
+                                         module)
+            }
+            if module == 'mobile/android/base':
+                paths['test'] = ['android-dtd']
+            projectconfig.add_paths(paths)
+            projectconfig.add_global_environment(l10n_base=self.l10nbase)
+        for child in aConfig.children:
+            self._config_for_ini(projectconfig, child)
 
 
 class EnumerateSourceTreeApp(EnumerateApp):
     '''Subclass EnumerateApp to work on side-by-side checked out
     repos, and to no pay attention to how the source would actually
     be checked out for building.
-
-    It's supporting applications like Fennec, too, which have
-    'locales/en-US/...' in their root dir, but claim to be 'mobile'.
     '''
 
     def __init__(self, inipath, basepath, l10nbase, redirects,
                  locales=None):
         self.basepath = basepath
         self.redirects = redirects
         EnumerateApp.__init__(self, inipath, l10nbase, locales)
 
     def setupConfigParser(self, inipath):
         self.config = SourceTreeConfigParser(inipath, self.basepath,
                                              self.redirects)
         self.config.loadConfigs()
-
-
-def get_base_path(mod, loc):
-    'statics for path patterns and conversion'
-    __l10n = 'l10n/%(loc)s/%(mod)s'
-    __en_US = 'mozilla/%(mod)s/locales/en-US'
-    if loc == 'en-US':
-        return __en_US % {'mod': mod}
-    return __l10n % {'mod': mod, 'loc': loc}
-
-
-def get_path(mod, loc, leaf):
-    return get_base_path(mod, loc) + '/' + leaf

new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/tests/test_apps.py
@@ -0,0 +1,116 @@
+import unittest
+import os
+import tempfile
+import shutil
+
+from compare_locales import mozpath
+from compare_locales.paths import EnumerateApp, ProjectFiles
+
+MAIL_INI = '''\
+[general]
+depth = ../..
+all = mail/locales/all-locales
+
+[compare]
+dirs = mail
+
+[includes]
+# non-central apps might want to use %(topsrcdir)s here, or other vars
+# RFE: that needs to be supported by compare-locales, too, though
+toolkit = mozilla/toolkit/locales/l10n.ini
+
+[include_toolkit]
+type = hg
+mozilla = mozilla-central
+repo = http://hg.mozilla.org/
+l10n.ini = toolkit/locales/l10n.ini
+'''
+
+
+MAIL_ALL_LOCALES = '''af
+de
+fr
+'''
+
+MAIL_FILTER_PY = '''
+def test(mod, path, entity = None):
+    if mod == 'toolkit' and path == 'ignored_path':
+        return 'ignore'
+    return 'error'
+'''
+
+TOOLKIT_INI = '''[general]
+depth = ../..
+
+[compare]
+dirs = toolkit
+'''
+
+
+class TestApp(unittest.TestCase):
+    def setUp(self):
+        self.stage = tempfile.mkdtemp()
+        mail = mozpath.join(self.stage, 'comm', 'mail', 'locales')
+        toolkit = mozpath.join(
+            self.stage, 'comm', 'mozilla', 'toolkit', 'locales')
+        l10n = mozpath.join(self.stage, 'l10n-central', 'de', 'toolkit')
+        os.makedirs(mozpath.join(mail, 'en-US'))
+        os.makedirs(mozpath.join(toolkit, 'en-US'))
+        os.makedirs(l10n)
+        with open(mozpath.join(mail, 'l10n.ini'), 'w') as f:
+            f.write(MAIL_INI)
+        with open(mozpath.join(mail, 'all-locales'), 'w') as f:
+            f.write(MAIL_ALL_LOCALES)
+        with open(mozpath.join(mail, 'filter.py'), 'w') as f:
+            f.write(MAIL_FILTER_PY)
+        with open(mozpath.join(toolkit, 'l10n.ini'), 'w') as f:
+            f.write(TOOLKIT_INI)
+        with open(mozpath.join(mail, 'en-US', 'mail.ftl'), 'w') as f:
+            f.write('')
+        with open(mozpath.join(toolkit, 'en-US', 'platform.ftl'), 'w') as f:
+            f.write('')
+        with open(mozpath.join(l10n, 'localized.ftl'), 'w') as f:
+            f.write('')
+
+    def tearDown(self):
+        shutil.rmtree(self.stage)
+
+    def test_app(self):
+        'Test parsing a App'
+        app = EnumerateApp(
+            mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
+            mozpath.join(self.stage, 'l10n-central'))
+        self.assertListEqual(app.locales, ['af', 'de', 'fr'])
+        self.assertEqual(len(app.config.children), 1)
+        projectconfig = app.asConfig()
+        self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])
+        files = ProjectFiles('de', [projectconfig])
+        files = list(files)
+        self.assertEqual(len(files), 3)
+
+        l10nfile, reffile, mergefile, test = files[0]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'mail', 'mail.ftl'])
+        self.assertListEqual(mozpath.split(reffile)[-4:],
+                             ['mail', 'locales', 'en-US', 'mail.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())
+
+        l10nfile, reffile, mergefile, test = files[1]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'toolkit', 'localized.ftl'])
+        self.assertListEqual(
+            mozpath.split(reffile)[-6:],
+            ['comm', 'mozilla', 'toolkit',
+             'locales', 'en-US', 'localized.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())
+
+        l10nfile, reffile, mergefile, test = files[2]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'toolkit', 'platform.ftl'])
+        self.assertListEqual(
+            mozpath.split(reffile)[-6:],
+            ['comm', 'mozilla', 'toolkit', 'locales', 'en-US', 'platform.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())

--- a/third_party/python/compare-locales/compare_locales/tests/test_checks.py
+++ b/third_party/python/compare-locales/compare_locales/tests/test_checks.py
@@ -1,42 +1,38 @@
 # -*- coding: utf-8 -*-
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 
 from compare_locales.checks import getChecker
-from compare_locales.parser import getParser, Parser, Entity
+from compare_locales.parser import getParser, Parser, DTDEntity
 from compare_locales.paths import File
 
 
 class BaseHelper(unittest.TestCase):
     file = None
     refContent = None
 
     def setUp(self):
         p = getParser(self.file.file)
         p.readContents(self.refContent)
         self.refList, self.refMap = p.parse()
 
-    def _test(self, content, refWarnOrErrors, with_ref_file=False):
+    def _test(self, content, refWarnOrErrors):
         p = getParser(self.file.file)
         p.readContents(content)
         l10n = [e for e in p]
         assert len(l10n) == 1
         l10n = l10n[0]
-        if with_ref_file:
-            kwargs = {
-                'reference': self.refList
-            }
-        else:
-            kwargs = {}
-        checker = getChecker(self.file, **kwargs)
+        checker = getChecker(self.file)
+        if checker.needs_reference:
+            checker.set_reference(self.refList)
         ref = self.refList[self.refMap[l10n.key]]
         found = tuple(checker.check(ref, l10n))
         self.assertEqual(found, refWarnOrErrors)
 
 
 class TestProperties(BaseHelper):
     file = File('foo.properties', 'foo.properties')
     refContent = '''some = value
@@ -179,90 +175,85 @@ class TestEntitiesInDTDs(BaseHelper):
     file = File('foo.dtd', 'foo.dtd')
     refContent = '''<!ENTITY short "This is &brandShortName;">
 <!ENTITY shorter "This is &brandShorterName;">
 <!ENTITY ent.start "Using &brandShorterName; start to">
 <!ENTITY ent.end " end">
 '''
 
     def testOK(self):
-        self._test('''<!ENTITY ent.start "Mit &brandShorterName;">''', tuple(),
-                   with_ref_file=True)
+        self._test('''<!ENTITY ent.start "Mit &brandShorterName;">''', tuple())
 
     def testMismatch(self):
         self._test('''<!ENTITY ent.start "Mit &brandShortName;">''',
                    (('warning', (0, 0),
                      'Entity brandShortName referenced, '
                      'but brandShorterName used in context',
-                     'xmlparse'),),
-                   with_ref_file=True)
+                     'xmlparse'),))
 
     def testAcross(self):
         self._test('''<!ENTITY ent.end "Mit &brandShorterName;">''',
-                   tuple(),
-                   with_ref_file=True)
+                   tuple())
 
     def testAcrossWithMismatch(self):
         '''If we could tell that ent.start and ent.end are one string,
         we should warn. Sadly, we can't, so this goes without warning.'''
         self._test('''<!ENTITY ent.end "Mit &brandShortName;">''',
-                   tuple(),
-                   with_ref_file=True)
+                   tuple())
 
     def testUnknownWithRef(self):
         self._test('''<!ENTITY ent.start "Mit &foopy;">''',
                    (('warning',
                      (0, 0),
                      'Referencing unknown entity `foopy` '
                      '(brandShorterName used in context, '
                      'brandShortName known)',
-                     'xmlparse'),),
-                   with_ref_file=True)
+                     'xmlparse'),))
 
     def testUnknown(self):
         self._test('''<!ENTITY ent.end "Mit &foopy;">''',
                    (('warning',
                      (0, 0),
                      'Referencing unknown entity `foopy`'
                      ' (brandShortName, brandShorterName known)',
-                     'xmlparse'),),
-                   with_ref_file=True)
+                     'xmlparse'),))
 
 
 class TestAndroid(unittest.TestCase):
     """Test Android checker
 
     Make sure we're hitting our extra rules only if
     we're passing in a DTD file in the embedding/android module.
     """
     apos_msg = u"Apostrophes in Android DTDs need escaping with \\' or " + \
                u"\\u0027, or use \u2019, or put string in quotes."
     quot_msg = u"Quotes in Android DTDs need escaping with \\\" or " + \
                u"\\u0022, or put string in apostrophes."
 
     def getEntity(self, v):
         ctx = Parser.Context(v)
-        return Entity(ctx, lambda s: s, '', (0, len(v)), (), (), (),
-                      (0, len(v)), ())
+        return DTDEntity(
+            ctx, '', (0, len(v)), (), (), (), (0, len(v)), ())
 
     def getDTDEntity(self, v):
         v = v.replace('"', '&quot;')
         ctx = Parser.Context('<!ENTITY foo "%s">' % v)
-        return Entity(ctx,
-                      lambda s: s, '',
-                      (0, len(v) + 16), (), (), (9, 12),
-                      (14, len(v) + 14), ())
+        return DTDEntity(
+            ctx,
+            '',
+            (0, len(v) + 16), (), (), (9, 12),
+            (14, len(v) + 14), ())
 
     def test_android_dtd(self):
         """Testing the actual android checks. The logic is involved,
         so this is a lot of nitty gritty detail tests.
         """
         f = File("embedding/android/strings.dtd", "strings.dtd",
                  "embedding/android")
-        checker = getChecker(f)
+        checker = getChecker(f, extra_tests=['android-dtd'])
         # good string
         ref = self.getDTDEntity("plain string")
         l10n = self.getDTDEntity("plain localized string")
         self.assertEqual(tuple(checker.check(ref, l10n)),
                          ())
         # dtd warning
         l10n = self.getDTDEntity("plain localized string &ref;")
         self.assertEqual(tuple(checker.check(ref, l10n)),
@@ -328,17 +319,17 @@ class TestAndroid(unittest.TestCase):
         l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+"  "+"\u0064")
         self.assertEqual(tuple(checker.check(ref, l10n)),
                          (('error', 14, 'truncated \\uXXXX escape',
                            'android'),))
 
     def test_android_prop(self):
         f = File("embedding/android/strings.properties", "strings.properties",
                  "embedding/android")
-        checker = getChecker(f)
+        checker = getChecker(f, extra_tests=['android-dtd'])
         # good plain string
         ref = self.getEntity("plain string")
         l10n = self.getEntity("plain localized string")
         self.assertEqual(tuple(checker.check(ref, l10n)),
                          ())
         # no dtd warning
         ref = self.getEntity("plain string")
         l10n = self.getEntity("plain localized string &ref;")
@@ -376,17 +367,18 @@ class TestAndroid(unittest.TestCase):
         self.assertEqual(tuple(checker.check(ref, l10n)),
                          ())
 
     def test_entities_across_dtd(self):
         f = File("browser/strings.dtd", "strings.dtd", "browser")
         p = getParser(f.file)
         p.readContents('<!ENTITY other "some &good.ref;">')
         ref = p.parse()
-        checker = getChecker(f, reference=ref[0])
+        checker = getChecker(f)
+        checker.set_reference(ref[0])
         # good string
         ref = self.getDTDEntity("plain string")
         l10n = self.getDTDEntity("plain localized string")
         self.assertEqual(tuple(checker.check(ref, l10n)),
                          ())
         # dtd warning
         ref = self.getDTDEntity("plain string")
         l10n = self.getDTDEntity("plain localized string &ref;")

--- a/third_party/python/compare-locales/compare_locales/tests/test_compare.py
+++ b/third_party/python/compare-locales/compare_locales/tests/test_compare.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 
-from compare_locales import compare
+from compare_locales import compare, paths
+from cPickle import loads, dumps
 
 
 class TestTree(unittest.TestCase):
     '''Test the Tree utility class
 
     Tree value classes need to be in-place editable
     '''
 
@@ -32,24 +33,20 @@ class TestTree(unittest.TestCase):
                 (1, 'value', {'leaf': 1}),
                 (0, 'key', ('two', 'other')),
                 (1, 'value', {'leaf': 2})
             ]
         )
         self.assertDictEqual(
             tree.toJSON(),
             {
-                'children': [
-                    ('one/entry',
-                     {'value': {'leaf': 1}}
-                     ),
-                    ('two/other',
-                     {'value': {'leaf': 2}}
-                     )
-                ]
+                'one/entry':
+                    {'leaf': 1},
+                'two/other':
+                    {'leaf': 2}
             }
         )
         self.assertMultiLineEqual(
             str(tree),
             '''\
 one/entry
     {'leaf': 1}
 two/other
@@ -69,22 +66,199 @@ two/other
                 (2, 'value', {'leaf': 1}),
                 (1, 'key', ('other',)),
                 (2, 'value', {'leaf': 2})
             ]
         )
         self.assertDictEqual(
             tree.toJSON(),
             {
-                'children': [
-                    ('one', {
-                        'children': [
-                            ('entry',
-                             {'value': {'leaf': 1}}
-                             ),
-                            ('other',
-                             {'value': {'leaf': 2}}
-                             )
-                        ]
-                    })
-                ]
+                'one': {
+                    'entry':
+                        {'leaf': 1},
+                    'other':
+                        {'leaf': 2}
+                }
             }
         )
+
+
+class TestObserver(unittest.TestCase):
+    def test_simple(self):
+        obs = compare.Observer()
+        f = paths.File('/some/real/sub/path', 'sub/path', locale='de')
+        obs.notify('missingEntity', f, 'one')
+        obs.notify('missingEntity', f, 'two')
+        obs.updateStats(f, {'missing': 15})
+        self.assertDictEqual(obs.toJSON(), {
+            'summary': {
+                'de': {
+                    'missing': 15
+                }
+            },
+            'details': {
+                'de/sub/path':
+                    [{'missingEntity': 'one'},
+                     {'missingEntity': 'two'}]
+            }
+        })
+        clone = loads(dumps(obs))
+        self.assertDictEqual(clone.summary, obs.summary)
+        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
+        self.assertIsNone(clone.file_stats)
+
+    def test_module(self):
+        obs = compare.Observer(file_stats=True)
+        f = paths.File('/some/real/sub/path', 'path',
+                       module='sub', locale='de')
+        obs.notify('missingEntity', f, 'one')
+        obs.notify('obsoleteEntity', f, 'bar')
+        obs.notify('missingEntity', f, 'two')
+        obs.updateStats(f, {'missing': 15})
+        self.assertDictEqual(obs.toJSON(), {
+            'summary': {
+                'de': {
+                    'missing': 15
+                }
+            },
+            'details': {
+                'de/sub/path':
+                    [
+                     {'missingEntity': 'one'},
+                     {'obsoleteEntity': 'bar'},
+                     {'missingEntity': 'two'},
+                    ]
+            }
+        })
+        self.assertDictEqual(obs.file_stats, {
+            'de': {
+                'sub/path': {
+                    'missing': 15
+                }
+            }
+        })
+        self.assertEqual(obs.serialize(), '''\
+de/sub/path
+    +one
+    -bar
+    +two
+de:
+missing: 15
+0% of entries changed''')
+        clone = loads(dumps(obs))
+        self.assertDictEqual(clone.summary, obs.summary)
+        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
+        self.assertDictEqual(clone.file_stats, obs.file_stats)
+
+    def test_file_stats(self):
+        obs = compare.Observer(file_stats=True)
+        f = paths.File('/some/real/sub/path', 'sub/path', locale='de')
+        obs.notify('missingEntity', f, 'one')
+        obs.notify('missingEntity', f, 'two')
+        obs.updateStats(f, {'missing': 15})
+        self.assertDictEqual(obs.toJSON(), {
+            'summary': {
+                'de': {
+                    'missing': 15
+                }
+            },
+            'details': {
+                'de/sub/path':
+                    [
+                     {'missingEntity': 'one'},
+                     {'missingEntity': 'two'},
+                    ]
+            }
+        })
+        self.assertDictEqual(obs.file_stats, {
+            'de': {
+                'sub/path': {
+                    'missing': 15
+                }
+            }
+        })
+        clone = loads(dumps(obs))
+        self.assertDictEqual(clone.summary, obs.summary)
+        self.assertDictEqual(clone.details.toJSON(), obs.details.toJSON())
+        self.assertDictEqual(clone.file_stats, obs.file_stats)
+
+
+class TestAddRemove(unittest.TestCase):
+
+    def _test(self, left, right, ref_actions):
+        ar = compare.AddRemove()
+        ar.set_left(left)
+        ar.set_right(right)
+        actions = list(ar)
+        self.assertListEqual(actions, ref_actions)
+
+    def test_equal(self):
+        self._test(['z', 'a', 'p'], ['z', 'a', 'p'], [
+                ('equal', 'z'),
+                ('equal', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_add_start(self):
+        self._test(['a', 'p'], ['z', 'a', 'p'], [
+                ('add', 'z'),
+                ('equal', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_add_middle(self):
+        self._test(['z', 'p'], ['z', 'a', 'p'], [
+                ('equal', 'z'),
+                ('add', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_add_end(self):
+        self._test(['z', 'a'], ['z', 'a', 'p'], [
+                ('equal', 'z'),
+                ('equal', 'a'),
+                ('add', 'p'),
+            ])
+
+    def test_delete_start(self):
+        self._test(['z', 'a', 'p'], ['a', 'p'], [
+                ('delete', 'z'),
+                ('equal', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_delete_middle(self):
+        self._test(['z', 'a', 'p'], ['z', 'p'], [
+                ('equal', 'z'),
+                ('delete', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_delete_end(self):
+        self._test(['z', 'a', 'p'], ['z', 'a'], [
+                ('equal', 'z'),
+                ('equal', 'a'),
+                ('delete', 'p'),
+            ])
+
+    def test_replace_start(self):
+        self._test(['b', 'a', 'p'], ['z', 'a', 'p'], [
+                ('add', 'z'),
+                ('delete', 'b'),
+                ('equal', 'a'),
+                ('equal', 'p'),
+            ])
+
+    def test_replace_middle(self):
+        self._test(['z', 'b', 'p'], ['z', 'a', 'p'], [
+                ('equal', 'z'),
+                ('add', 'a'),
+                ('delete', 'b'),
+                ('equal', 'p'),
+            ])
+
+    def test_replace_end(self):
+        self._test(['z', 'a', 'b'], ['z', 'a', 'p'], [
+                ('equal', 'z'),
+                ('equal', 'a'),
+                ('add', 'p'),
+                ('delete', 'b'),
+            ])

new file mode 100644
--- /dev/null
+++ b/third_party/python/compare-locales/compare_locales/tests/test_defines.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from compare_locales.tests import ParserTestMixin
+
+
+mpl2 = '''\
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+'''
+
+
+class TestDefinesParser(ParserTestMixin, unittest.TestCase):
+
+    filename = 'defines.inc'
+
+    def testBrowser(self):
+        self._test(mpl2 + '''#filter emptyLines
+
+#define MOZ_LANGPACK_CREATOR mozilla.org
+
+# If non-English locales wish to credit multiple contributors, uncomment this
+# variable definition and use the format specified.
+# #define MOZ_LANGPACK_CONTRIBUTORS <em:contributor>Joe Solon</em:contributor>
+
+#unfilter emptyLines
+
+''', (
+            ('Comment', mpl2),
+            ('DefinesInstruction', 'filter emptyLines'),
+            ('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
+            ('Comment', '#define'),
+            ('DefinesInstruction', 'unfilter emptyLines')))
+
+    def testBrowserWithContributors(self):
+        self._test(mpl2 + '''#filter emptyLines
+
+#define MOZ_LANGPACK_CREATOR mozilla.org
+
+# If non-English locales wish to credit multiple contributors, uncomment this
+# variable definition and use the format specified.
+#define MOZ_LANGPACK_CONTRIBUTORS <em:contributor>Joe Solon</em:contributor>
+
+#unfilter emptyLines
+
+''', (
+            ('Comment', mpl2),
+            ('DefinesInstruction', 'filter emptyLines'),
+            ('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
+            ('Comment', 'non-English'),
+            ('MOZ_LANGPACK_CONTRIBUTORS',
+             '<em:contributor>Joe Solon</em:contributor>'),
+            ('DefinesInstruction', 'unfilter emptyLines')))
+
+    def testCommentWithNonAsciiCharacters(self):
+        self._test(mpl2 + '''#filter emptyLines
+
+# e.g. #define seamonkey_l10n <DT><A HREF="urn:foo">SeaMonkey v češtině</a>
+#define seamonkey_l10n_long
+
+#unfilter emptyLines
+
+''', (
+            ('Comment', mpl2),
+            ('DefinesInstruction', 'filter emptyLines'),
+            ('Comment', u'češtině'),
+            ('seamonkey_l10n_long', ''),
+            ('DefinesInstruction', 'unfilter emptyLines')))
+
+    def testToolkit(self):
+        self._test('''#define MOZ_LANG_TITLE English (US)
+''', (
+            ('MOZ_LANG_TITLE', 'English (US)'),))
+
+    def testToolkitEmpty(self):
+        self._test('', tuple())
+
+    def test_empty_file(self):
+        '''Test that empty files generate errors
+
+        defines.inc are interesting that way, as their
+        content is added to the generated file.
+        '''
+        self._test('\n', (('Junk', '\n'),))
+        self._test('\n\n', (('Junk', '\n\n'),))
+        self._test(' \n\n', (('Junk', ' \n\n'),))
+
+
+if __name__ == '__main__':
+    unittest.main()