bug 1370176, generic configuration for l10n, r?flod,gps draft
authorAxel Hecht <axel@pike.org>
Mon, 05 Jun 2017 10:28:33 +0200
changeset 591955 6ffdd0c0dacdc49a594c6432e9b2d0e2046080f2
parent 591727 eca8d0ea03af1d2424550a037f714f14c0f7b1be
child 632664 4821e04463c75d7be4cc736592efdfaadf4d337b
push id63217
push useraxel@mozilla.com
push dateFri, 09 Jun 2017 18:04:08 +0000
reviewersflod, gps
bugs1370176
milestone55.0a1
bug 1370176, generic configuration for l10n, r?flod,gps The current way to configure compare-locales has a lot of assumptions that make our l10n system really stubborn. The generic configuration is independent of python, and uses toml files for configuration. They're still modular, but there's only one file format. See http://moz-l10n-config.readthedocs.io/en/latest/fileformat.html for the specification. Also fixes a few nits in filter.py, where we compared the entity key as bool, which is false if we pass in ''. Explicitly compare as "entity is None" to be precise about when we're checking files. MozReview-Commit-ID: 5TmfobaImF4
browser/locales/filter.py
browser/locales/l10n.toml
devtools/client/locales/l10n.toml
devtools/shared/locales/l10n.toml
mobile/android/locales/filter.py
mobile/android/locales/jar.mn
mobile/android/locales/l10n.toml
moz.build
toolkit/locales/l10n.toml
tools/compare-locales/docs/index.rst
--- a/browser/locales/filter.py
+++ b/browser/locales/filter.py
@@ -13,17 +13,17 @@ def test(mod, path, entity = None):
                  "extensions/spellcheck",
                  "other-licenses/branding/firefox",
                  "browser/branding/official",
                  "services/sync"):
     return "ignore"
   if mod not in ("browser", "extensions/spellcheck"):
     # we only have exceptions for browser and extensions/spellcheck
     return "error"
-  if not entity:
+  if entity is None:
     # the only files to ignore are spell checkers
     if mod == "extensions/spellcheck":
       return "ignore"
     return "error"
   if mod == "extensions/spellcheck":
     # l10n ships en-US dictionary or something, do compare
     return "error"
   if path == "defines.inc":
new file mode 100644
--- /dev/null
+++ b/browser/locales/l10n.toml
@@ -0,0 +1,169 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../.."
+
+locales = [
+    "ach",
+    "af",
+    "an",
+    "ar",
+    "as",
+    "ast",
+    "az",
+    "be",
+    "bg",
+    "bn-BD",
+    "bn-IN",
+    "br",
+    "bs",
+    "ca",
+    "cak",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dsb",
+    "el",
+    "en-GB",
+    "en-ZA",
+    "eo",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et",
+    "eu",
+    "fa",
+    "ff",
+    "fi",
+    "fr",
+    "fy-NL",
+    "ga-IE",
+    "gd",
+    "gl",
+    "gn",
+    "gu-IN",
+    "he",
+    "hi-IN",
+    "hr",
+    "hsb",
+    "hu",
+    "hy-AM",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "ja-JP-mac",
+    "ka",
+    "kab",
+    "kk",
+    "km",
+    "kn",
+    "ko",
+    "lij",
+    "lo",
+    "lt",
+    "ltg",
+    "lv",
+    "mai",
+    "mk",
+    "ml",
+    "mr",
+    "ms",
+    "my",
+    "nb-NO",
+    "ne-NP",
+    "nl",
+    "nn-NO",
+    "or",
+    "pa-IN",
+    "pl",
+    "pt-BR",
+    "pt-PT",
+    "rm",
+    "ro",
+    "ru",
+    "si",
+    "sk",
+    "sl",
+    "son",
+    "sq",
+    "sr",
+    "sv-SE",
+    "ta",
+    "te",
+    "th",
+    "tl",
+    "tr",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "xh",
+    "zh-CN",
+    "zh-TW",
+    ]
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "browser/locales/en-US/**"
+    l10n = "{l}browser/**"
+
+[[paths]]
+    reference = "browser/branding/official/locales/en-US/**"
+    l10n = "{l}browser/branding/official/**"
+
+[[paths]]
+    reference = "browser/extensions/onboarding/locales/en-US/**"
+    l10n = "{l}browser/extensions/onboarding/**"
+
+[[paths]]
+    reference = "browser/extensions/webcompat-reporter/locales/en-US/**"
+    l10n = "{l}browser/extensions/webcompat-reporter/**"
+
+[[paths]]
+    reference = "services/sync/locales/en-US/**"
+    l10n = "{l}services/sync/**"
+
+
+[[includes]]
+    path = "toolkit/locales/l10n.toml"
+
+[[includes]]
+    path = "devtools/client/locales/l10n.toml"
+
+# Filters
+# The filters below are evaluated one after the other, in the given order.
+# Enter a combination of path as in the localization, key, and an action,
+# to change the default behavior of compare-locales and l10n merge.
+#
+# For browser/locales/en-US/chrome/browser/foo.properties,
+# path would be {l}browser/chrome/browser/foo.properties
+# key: the key/id of the entity
+# If key isn't specified, the complete file can be missing.
+[[filters]]
+    path = "{l}browser/defines.inc"
+    key = "MOZ_LANGPACK_CONTRIBUTORS"
+    action = "ignore"
+
+[[filters]]
+    path = [
+        "{l}browser/defines.inc",
+        "{l}browser/firefox-l10n.js",
+    ]
+    action = "ignore"
+
+[[filters]]
+    path = "{l}browser/chrome/browser-region/region.properties"
+    key = [
+        "re:^browser\\.search\\.order\\.[1-9]$",
+        "re:^browser\\.contentHandlers\\.types\\.[0-5]\\..*$",
+        "re:^gecko\\.handlerService\\.schemes\\..*$",
+        "re:^gecko\\.handlerService\\.defaultHandlersVersion$"
+    ]
+    action = "ignore"
new file mode 100644
--- /dev/null
+++ b/devtools/client/locales/l10n.toml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+[[paths]]
+    reference = "devtools/client/locales/en-US/**"
+    l10n = "{l}devtools/client/**"
new file mode 100644
--- /dev/null
+++ b/devtools/shared/locales/l10n.toml
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+[[paths]]
+    reference = "devtools/shared/locales/en-US/**"
+    l10n = "{l}devtools/shared/**"
--- a/mobile/android/locales/filter.py
+++ b/mobile/android/locales/filter.py
@@ -48,17 +48,17 @@ def test(mod, path, entity = None):
         "chrome/plugins.properties"):
       return "error"
     return "ignore"
 
   if mod not in ("mobile", "mobile/android"):
     # we only have exceptions for mobile*
     return "error"
   if mod == "mobile/android":
-    if not entity:
+    if entity is None:
       if (re.match(r"mobile-l10n.js", path) or
           re.match(r"defines.inc", path)):
         return "ignore"
     if path == "defines.inc":
       if entity == "MOZ_LANGPACK_CONTRIBUTORS":
         return "ignore"
     return "error"
 
--- a/mobile/android/locales/jar.mn
+++ b/mobile/android/locales/jar.mn
@@ -32,17 +32,17 @@
   locale/@AB_CD@/browser/aboutLogins.dtd          (%chrome/aboutLogins.dtd)
   locale/@AB_CD@/browser/aboutLogins.properties  (%chrome/aboutLogins.properties)
 #ifndef RELEASE_OR_BETA
   locale/@AB_CD@/browser/webcompatReporter.properties (%chrome/webcompatReporter.properties)
 #endif
 % resource search-plugins chrome://browser/locale/searchplugins/
 
 # overrides for toolkit l10n, also for en-US
-# keep this file list in sync with filter.py
+# keep this file list in sync with l10n.toml and filter.py
 relativesrcdir toolkit/locales:
   locale/@AB_CD@/browser/overrides/about.dtd                       (%chrome/global/about.dtd)
   locale/@AB_CD@/browser/overrides/aboutAbout.dtd                  (%chrome/global/aboutAbout.dtd)
   locale/@AB_CD@/browser/overrides/aboutReader.properties          (%chrome/global/aboutReader.properties)
   locale/@AB_CD@/browser/overrides/aboutRights.dtd                 (%chrome/global/aboutRights.dtd)
   locale/@AB_CD@/browser/overrides/charsetMenu.properties          (%chrome/global/charsetMenu.properties)
   locale/@AB_CD@/browser/overrides/commonDialogs.properties        (%chrome/global/commonDialogs.properties)
   locale/@AB_CD@/browser/overrides/intl.properties                 (%chrome/global/intl.properties)
new file mode 100644
--- /dev/null
+++ b/mobile/android/locales/l10n.toml
@@ -0,0 +1,246 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+locales = [
+    "an",
+    "ar",
+    "as",
+    "ast",
+    "az",
+    "be",
+    "bg",
+    "bn-BD",
+    "bn-IN",
+    "br",
+    "ca",
+    "cak",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dsb",
+    "el",
+    "en-GB",
+    "en-ZA",
+    "eo",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et",
+    "eu",
+    "fa",
+    "ff",
+    "fi",
+    "fr",
+    "fy-NL",
+    "ga-IE",
+    "gd",
+    "gl",
+    "gn",
+    "gu-IN",
+    "he",
+    "hi-IN",
+    "hr",
+    "hsb",
+    "hu",
+    "hy-AM",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "ka",
+    "kab",
+    "kk",
+    "kn",
+    "ko",
+    "lo",
+    "lt",
+    "lv",
+    "mai",
+    "ml",
+    "mr",
+    "ms",
+    "my",
+    "nb-NO",
+    "ne-NP",
+    "nl",
+    "nn-NO",
+    "or",
+    "pa-IN",
+    "pl",
+    "pt-BR",
+    "pt-PT",
+    "rm",
+    "ro",
+    "ru",
+    "sk",
+    "sl",
+    "son",
+    "sq",
+    "sr",
+    "sv-SE",
+    "ta",
+    "te",
+    "th",
+    "tr",
+    "trs",
+    "uk",
+    "ur",
+    "uz",
+    "wo",
+    "xh",
+    "zam",
+    "zh-CN",
+    "zh-TW",
+    ]
+
+[build]
+exclude-multi-locale = [
+    "be",
+    "bn-BD",
+    "ne-NP",
+    "trs",
+    "wo",
+    "zam",
+]
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "mobile/locales/en-US/**"
+    l10n = "{l}mobile/**"
+
+[[paths]]
+    reference = "mobile/android/locales/en-US/**"
+    l10n = "{l}mobile/android/**"
+
+[[paths]]
+    reference = "mobile/android/base/locales/en-US/**"
+    l10n = "{l}mobile/android/base/**"
+    test = [
+        "android-dtd",
+    ]
+
+# hand-picked paths from toolkit, keep in sync with jar.mn
+[[paths]]
+    reference = "dom/locales/en-US/chrome/global.dtd"
+    l10n = "{l}dom/chrome/global.dtd"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/accessibility/AccessFu.properties"
+    l10n = "{l}dom/chrome/accessibility/AccessFu.properties"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/dom/dom.properties"
+    l10n = "{l}dom/chrome/dom/dom.properties"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/plugins.properties"
+    l10n = "{l}dom/chrome/plugins.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/about.dtd"
+    l10n = "{l}toolkit/chrome/global/about.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutAbout.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutAbout.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutReader.properties"
+    l10n = "{l}toolkit/chrome/global/aboutReader.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutRights.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutRights.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/charsetMenu.properties"
+    l10n = "{l}toolkit/chrome/global/charsetMenu.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/commonDialogs.properties"
+    l10n = "{l}toolkit/chrome/global/commonDialogs.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/intl.properties"
+    l10n = "{l}toolkit/chrome/global/intl.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/intl.css"
+    l10n = "{l}toolkit/chrome/global/intl.css"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/passwordmgr/passwordmgr.properties"
+    l10n = "{l}toolkit/chrome/passwordmgr/passwordmgr.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/search/search.properties"
+    l10n = "{l}toolkit/chrome/search/search.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/pluginproblem/pluginproblem.dtd"
+    l10n = "{l}toolkit/chrome/pluginproblem/pluginproblem.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutSupport.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutSupport.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutSupport.properties"
+    l10n = "{l}toolkit/chrome/global/aboutSupport.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/crashreporter/crashes.dtd"
+    l10n = "{l}toolkit/crashreporter/crashes.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/crashreporter/crashes.properties"
+    l10n = "{l}toolkit/crashreporter/crashes.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/mozilla.dtd"
+    l10n = "{l}toolkit/chrome/global/mozilla.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutTelemetry.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutTelemetry.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutTelemetry.properties"
+    l10n = "{l}toolkit/chrome/global/aboutTelemetry.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutWebrtc.properties"
+    l10n = "{l}toolkit/chrome/global/aboutWebrtc.properties"
+
+
+[[filters]]
+    path = [
+        "{l}mobile/android/mobile-l10n.js",
+        "{l}mobile/android/defines.inc",
+    ]
+    action = "ignore"
+
+[[filters]]
+    path = "{l}mobile/android/defines.inc"
+    key = "MOZ_LANGPACK_CONTRIBUTORS"
+    action = "ignore"
+
+[[filters]]
+    path = "{l}mobile/chrome/region.properties"
+    key = [
+        "re:^browser\\.search\\.order\\.[1-9]$",
+        "re:^browser\\.search\\.[a-zA-Z]+\\.US",
+        "re:^browser\\.contentHandlers\\.types\\.[0-5]\\..*$",
+        "re:^gecko\\.handlerService\\.schemes\\..+$",
+        "re:^gecko\\.handlerService\\.defaultHandlersVersion$",
+        "re:^browser\\.suggestedsites\\..+$",
+    ]
+    action = "ignore"
--- a/moz.build
+++ b/moz.build
@@ -35,16 +35,20 @@ with Files('*moz*'):
     BUG_COMPONENT = ('Core', 'Build Config')
 
 with Files('GNUmakefile'):
     BUG_COMPONENT = ('Core', 'Build Config')
 
 with Files('*gradle*'):
     BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support')
 
+with Files('**/l10n.toml'):
+    BUG_COMPONENT = ('Core', 'Localization')
+    FINAL = True
+
 with Files('README.txt'):
     BUG_COMPONENT = ('Core', 'General')
 
 with Files('**/Makefile.in'):
     BUG_COMPONENT = ('Core', 'Build Config')
     FINAL = True
 
 FILES_PER_UNIFIED_FILE = 1
new file mode 100644
--- /dev/null
+++ b/toolkit/locales/l10n.toml
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../.."
+
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "toolkit/locales/en-US/**"
+    l10n = "{l}toolkit/**"
+
+[[paths]]
+    reference = "dom/locales/en-US/**"
+    l10n = "{l}dom/**"
+
+[[paths]]
+    reference = "netwerk/locales/en-US/**"
+    l10n = "{l}netwerk/**"
+
+[[paths]]
+    reference = "security/manager/locales/en-US/**"
+    l10n = "{l}security/manager/**"
+
+[[includes]]
+    path = "devtools/shared/locales/l10n.toml"
--- a/tools/compare-locales/docs/index.rst
+++ b/tools/compare-locales/docs/index.rst
@@ -15,17 +15,34 @@ idea to check the :doc:`./glossary` for 
 we use for what.
 
 Exposing strings
 ----------------
 
 Localizers only handle a few file formats in well-known locations in the
 source tree.
 
-The locations are in directories like
+The locations are specified by TOML files. They're part of the bigger
+localization ecosystem at Mozilla, and `the documentation about the
+file format <http://moz-l10n-config.readthedocs.io/en/latest/fileformat.html>`_
+explains how to set them up, and what the entries mean. In short, you find
+
+.. code-block:: toml
+
+    [[paths]]
+        reference = browser/locales/en-US/**
+        l10n = {l}browser/**
+
+to add a directory for all localizations. Changes to these files are best
+submitted for review by :Pike or :flod.
+
+These configuration files are the future, and right now, we still have
+support for the previous way to configuring l10n, which is described below.
+
+The locations are commonly in directories like
 
     :file:`browser/`\ ``locales/en-US/``\ :file:`subdir/file.ext`
 
 The first thing to note is that only files beneath :file:`locales/en-US` are
 exposed to localizers. The second thing to note is that only a few directories
 are exposed. Which directories are exposed is defined in files called
 ``l10n.ini``, which are at a
 `few places <https://dxr.mozilla.org/mozilla-central/search?q=path%3Al10n.ini&redirect=true>`_
@@ -83,17 +100,17 @@ The following file formats are known to 
 DTD
     Used in XUL and XHTML. Also for Android native strings.
 Properties
     Used from JavaScript and C++. When used from js, also comes with
     `plural support <https://developer.mozilla.org/docs/Mozilla/Localization/Localization_and_Plurals>`_.
 ini
     Used by the crashreporter and updater, avoid if possible.
 foo.defines
-    Used during builds, for example to create file:`install.rdf` for
+    Used during builds, for example to create :file:`install.rdf` for
     language packs.
 
 Adding new formats involves changing various different tools, and is strongly
 discouraged.
 
 Exceptions
 ----------
 Generally, anything that exists in ``en-US`` needs a one-to-one mapping in