bug 1353680, create test to prevent bad content in localizable strings
TBD: Still need to figure out where to put the whitelist of strings
that are actually OK to exist in different versions.
MozReview-Commit-ID: Jrw8f2YpPH
--- a/python/mozlint/mozlint/pathutils.py
+++ b/python/mozlint/mozlint/pathutils.py
@@ -10,16 +10,18 @@ from mozpack import path as mozpath
from mozpack.files import FileFinder
class FilterPath(object):
"""Helper class to make comparing and matching file paths easier."""
def __init__(self, path, exclude=None):
self.path = os.path.normpath(path)
self._finder = None
+ if exclude is None:
+ exclude = []
self.exclude = exclude
@property
def finder(self):
if self._finder:
return self._finder
self._finder = FileFinder(
mozpath.normsep(self.path),
new file mode 100644
--- /dev/null
+++ b/tools/lint/l10n.yml
@@ -0,0 +1,28 @@
+---
+l10n:
+ description: Localization linter
+ # list of include directories of both
+ # browser and mobile/android l10n.tomls
+ include:
+ - browser/locales/en-US
+ - browser/branding/official/locales/en-US
+ - browser/extensions/onboarding/locales/en-US
+ - browser/extensions/webcompat-reporter/locales/en-US
+ - devtools/client/locales/en-US
+ - devtools/shared/locales/en-US
+ - devtools/shim/locales/en-US
+ - dom/locales/en-US
+ - netwerk/locales/en-US
+ - security/manager/locales/en-US
+ - services/sync/locales/en-US
+ - toolkit/locales/en-US
+ # files not supported by compare-locales,
+ # and also not relevant to this linter
+ exclude:
+ - browser/locales/en-US/firefox-l10n.js
+ - toolkit/locales/en-US/chrome/global/intl.css
+ l10n_configs:
+ - browser/locales/l10n.toml
+ - mobile/android/locales/l10n.toml
+ type: external
+ payload: python.l10n_lint:lint
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/l10n_lint.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+
+from mozboot import util as mb_util
+from mozlint import result, pathutils
+from mozpack import path as mozpath
+import mozversioncontrol.repoupdate
+
+from compare_locales import compare, parser
+from compare_locales.paths import File, TOMLParser, ProjectFiles
+
+
+LOCALE = 'gecko-strings' # TODO: gecko-strings
+
+
+def ensure_gecko_strings(skip_clone):
+ state_dir, _ = mb_util.get_state_dir()
+ if skip_clone:
+ return state_dir
+ hg = mozversioncontrol.get_tool_path('hg')
+ mozversioncontrol.repoupdate.update_mercurial_repo(
+ hg,
+ 'https://hg.mozilla.org/users/axel_mozilla.com/gecko-strings',
+ mozpath.join(state_dir, 'gecko-strings'))
+ print("You can pass --cache-gecko-strings to the l10n linter\n"
+ "to reuse a current gecko-strings clone.\n\n"
+ "Update the clone at least daily to get reliable results.\n")
+ return state_dir
+
+
+def lint(paths, config, **lintargs):
+ if 'l10n_configs' not in config:
+ return [
+ result.from_config(config,
+ path=config['path'],
+ message='l10n linter not properly configured')]
+ extra_args = lintargs['extra_args']
+ skip_clone = bool(extra_args and '--cache-gecko-strings' in extra_args)
+ l10n_base = ensure_gecko_strings(skip_clone)
+ root = lintargs['root']
+ configs = []
+ for toml in config['l10n_configs']:
+ cfg = TOMLParser.parse(mozpath.join(root, toml),
+ ignore_missing_includes=True)
+ cfg.set_locales([LOCALE], deep=True)
+ cfg.add_global_environment(l10n_base=l10n_base)
+ configs.append(cfg)
+ files = ProjectFiles(LOCALE, configs)
+ all_files = []
+ for p in paths:
+ fp = pathutils.FilterPath(p)
+ if fp.isdir:
+ for _, fileobj in fp.finder:
+ all_files.append(fileobj.path)
+ if fp.isfile:
+ all_files.append(p)
+ # filter again, our directories might have picked up files we ignore
+ all_files = pathutils.filterpaths(all_files, config, **lintargs)
+ obs = compare.Observer(file_stats=True)
+ comparer = compare.ContentComparer([obs])
+ results = []
+ for path in all_files:
+ f = File(path, path, locale='en-US')
+ l10n, ref, _, extra_tests = files.match(path)
+ # check for errors
+ comparer.compare(f, f, None, extra_tests)
+ if not os.path.isfile(l10n):
+ continue
+ # check for changed strings
+ try:
+ file_parser = parser.getParser(ref)
+ except UserWarning:
+ results.append(result.from_config(
+ config,
+ level='warning',
+ path=path,
+ message="compare-locales doesn't support file format"
+ ))
+ continue
+ file_parser.readFile(ref)
+ ref_entities, ref_map = file_parser.parse()
+ file_parser.readFile(l10n)
+ l10n_entities, l10n_map = file_parser.parse()
+ for key in ref_map.keys():
+ if key not in l10n_map:
+ continue
+ ref_e = ref_entities[ref_map[key]]
+ l10n_e = l10n_entities[l10n_map[key]]
+ if not ref_e.equals(l10n_e):
+ lineno, col = ref_e.position()
+ res = {
+ 'path': ref,
+ 'lineno': lineno,
+ 'column': col,
+ 'message': 'Change ID for changed string: {}'.format(key)
+ }
+ results.append(result.from_config(config, **res))
+
+ last_depth = last_paths = None
+ for depth, kind, data in obs.details.getContent():
+ if kind == 'value':
+ p = '/'.join(last_paths)
+ for entry in data:
+ if 'error' in entry:
+ results.append(result.from_config(config, path=p, message=entry['error']))
+ continue
+ if last_paths is None:
+ last_paths = data[1:] # skip en-US
+ else:
+ if depth <= last_depth:
+ last_paths = last_paths[:-(last_depth-depth+1)]
+ last_paths = last_paths + data
+ last_depth = depth
+ return results