bug 1353680, create test to prevent bad content in localizable strings draft
authorAxel Hecht <axel@pike.org>
Thu, 05 Oct 2017 19:05:34 +0200
changeset 675765 1faaf85f18a422eedadf67b120356e4cbe832f04
parent 675508 53bbdaaa2b8c1819061be26101b075c081b23260
child 675766 f6f3bc2cf38987f45eeeb4ea68a2063295eac6c7
push id83236
push useraxel@mozilla.com
push dateThu, 05 Oct 2017 21:08:56 +0000
bugs1353680
milestone58.0a1
bug 1353680, create test to prevent bad content in localizable strings TBD: Still need to figure out where to put the whitelist of strings that are actually OK to exist in different versions. MozReview-Commit-ID: Jrw8f2YpPH
python/mozlint/mozlint/pathutils.py
tools/lint/l10n.yml
tools/lint/python/l10n_lint.py
--- a/python/mozlint/mozlint/pathutils.py
+++ b/python/mozlint/mozlint/pathutils.py
@@ -10,16 +10,18 @@ from mozpack import path as mozpath
 from mozpack.files import FileFinder
 
 
 class FilterPath(object):
     """Helper class to make comparing and matching file paths easier."""
     def __init__(self, path, exclude=None):
         self.path = os.path.normpath(path)
         self._finder = None
+        if exclude is None:
+            exclude = []
         self.exclude = exclude
 
     @property
     def finder(self):
         if self._finder:
             return self._finder
         self._finder = FileFinder(
             mozpath.normsep(self.path),
new file mode 100644
--- /dev/null
+++ b/tools/lint/l10n.yml
@@ -0,0 +1,28 @@
+---
+l10n:
+    description: Localization linter
+    # list of include directories of both
+    # browser and mobile/android l10n.tomls
+    include:
+        - browser/locales/en-US
+        - browser/branding/official/locales/en-US
+        - browser/extensions/onboarding/locales/en-US
+        - browser/extensions/webcompat-reporter/locales/en-US
+        - devtools/client/locales/en-US
+        - devtools/shared/locales/en-US
+        - devtools/shim/locales/en-US
+        - dom/locales/en-US
+        - netwerk/locales/en-US
+        - security/manager/locales/en-US
+        - services/sync/locales/en-US
+        - toolkit/locales/en-US
+    # files not supported by compare-locales,
+    # and also not relevant to this linter
+    exclude:
+        - browser/locales/en-US/firefox-l10n.js
+        - toolkit/locales/en-US/chrome/global/intl.css
+    l10n_configs:
+        - browser/locales/l10n.toml
+        - mobile/android/locales/l10n.toml
+    type: external
+    payload: python.l10n_lint:lint
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/l10n_lint.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+
+from mozboot import util as mb_util
+from mozlint import result, pathutils
+from mozpack import path as mozpath
+import mozversioncontrol.repoupdate
+
+from compare_locales import compare, parser
+from compare_locales.paths import File, TOMLParser, ProjectFiles
+
+
+LOCALE = 'gecko-strings'  # TODO: gecko-strings
+
+
+def ensure_gecko_strings(skip_clone):
+    state_dir, _ = mb_util.get_state_dir()
+    if skip_clone:
+        return state_dir
+    hg = mozversioncontrol.get_tool_path('hg')
+    mozversioncontrol.repoupdate.update_mercurial_repo(
+        hg,
+        'https://hg.mozilla.org/users/axel_mozilla.com/gecko-strings',
+        mozpath.join(state_dir, 'gecko-strings'))
+    print("You can pass --cache-gecko-strings to the l10n linter\n"
+          "to reuse a current gecko-strings clone.\n\n"
+          "Update the clone at least daily to get reliable results.\n")
+    return state_dir
+
+
+def lint(paths, config, **lintargs):
+    if 'l10n_configs' not in config:
+        return [
+            result.from_config(config,
+                               path=config['path'],
+                               message='l10n linter not properly configured')]
+    extra_args = lintargs['extra_args']
+    skip_clone = bool(extra_args and '--cache-gecko-strings' in extra_args)
+    l10n_base = ensure_gecko_strings(skip_clone)
+    root = lintargs['root']
+    configs = []
+    for toml in config['l10n_configs']:
+        cfg = TOMLParser.parse(mozpath.join(root, toml),
+                               ignore_missing_includes=True)
+        cfg.set_locales([LOCALE], deep=True)
+        cfg.add_global_environment(l10n_base=l10n_base)
+        configs.append(cfg)
+    files = ProjectFiles(LOCALE, configs)
+    all_files = []
+    for p in paths:
+        fp = pathutils.FilterPath(p)
+        if fp.isdir:
+            for _, fileobj in fp.finder:
+                all_files.append(fileobj.path)
+        if fp.isfile:
+            all_files.append(p)
+    # filter again, our directories might have picked up files we ignore
+    all_files = pathutils.filterpaths(all_files, config, **lintargs)
+    obs = compare.Observer(file_stats=True)
+    comparer = compare.ContentComparer([obs])
+    results = []
+    for path in all_files:
+        f = File(path, path, locale='en-US')
+        l10n, ref, _, extra_tests = files.match(path)
+        # check for errors
+        comparer.compare(f, f, None, extra_tests)
+        if not os.path.isfile(l10n):
+            continue
+        # check for changed strings
+        try:
+            file_parser = parser.getParser(ref)
+        except UserWarning:
+            results.append(result.from_config(
+                config,
+                level='warning',
+                path=path,
+                message="compare-locales doesn't support file format"
+                ))
+            continue
+        file_parser.readFile(ref)
+        ref_entities, ref_map = file_parser.parse()
+        file_parser.readFile(l10n)
+        l10n_entities, l10n_map = file_parser.parse()
+        for key in ref_map.keys():
+            if key not in l10n_map:
+                continue
+            ref_e = ref_entities[ref_map[key]]
+            l10n_e = l10n_entities[l10n_map[key]]
+            if not ref_e.equals(l10n_e):
+                lineno, col = ref_e.position()
+                res = {
+                    'path': ref,
+                    'lineno': lineno,
+                    'column': col,
+                    'message': 'Change ID for changed string: {}'.format(key)
+                }
+                results.append(result.from_config(config, **res))
+
+    last_depth = last_paths = None
+    for depth, kind, data in obs.details.getContent():
+        if kind == 'value':
+            p = '/'.join(last_paths)
+            for entry in data:
+                if 'error' in entry:
+                    results.append(result.from_config(config, path=p, message=entry['error']))
+            continue
+        if last_paths is None:
+            last_paths = data[1:]  # skip en-US
+        else:
+            if depth <= last_depth:
+                last_paths = last_paths[:-(last_depth-depth+1)]
+            last_paths = last_paths + data
+        last_depth = depth
+    return results