Bug 1395459 - Store the version of the locale data in a langpack based on HG push timestamp. r?gps draft
authorZibi Braniecki <zbraniecki@mozilla.com>
Sun, 10 Sep 2017 19:57:34 -0700
changeset 670526 d1324a09028f207d783547a6f7f63c3cc6428bfa
parent 669596 7e962631ba4298bcefa571008661983d77c3e652
child 733254 d00363c77a4997f0015d05a0aa0b2ab0b74c0504
push id81652
push userbmo:gandalf@aviary.pl
push dateTue, 26 Sep 2017 15:04:43 +0000
reviewersgps
bugs1395459
milestone58.0a1
Bug 1395459 - Store the version of the locale data in a langpack based on HG push timestamp. r?gps MozReview-Commit-ID: ELx9VW81s6Q
python/mozbuild/mozbuild/action/langpack_manifest.py
toolkit/locales/l10n.mk
--- a/python/mozbuild/mozbuild/action/langpack_manifest.py
+++ b/python/mozbuild/mozbuild/action/langpack_manifest.py
@@ -10,29 +10,101 @@
 ###
 from __future__ import absolute_import
 
 import argparse
 import sys
 import os
 import json
 import io
+import datetime
+import requests
+import mozversioncontrol
 from mozpack.chrome.manifest import (
     Manifest,
     ManifestLocale,
     parse_manifest,
 )
 from mozbuild.preprocessor import Preprocessor
 
 
 def write_file(path, content):
     with io.open(path, 'w', encoding='utf-8') as out:
         out.write(content + '\n')
 
 
+pushlog_api_url = "{0}/json-rev/{1}"
+
+
+###
+# Retrievers a UTC datetime of the push for the current commit
+# from a mercurial clone directory.
+#
+# Args:
+#    path (str) - path to a directory
+#
+# Returns:
+#    (datetime) - a datetime object
+#
+# Example:
+#    dt = get_dt_from_hg("/var/vcs/l10n-central/pl")
+#    dt == datetime(2017, 10, 11, 23, 31, 54, 0)
+###
+def get_dt_from_hg(path):
+    with mozversioncontrol.get_repository_object(path=path) as repo:
+        repo_url = repo._run_in_client(["paths", "default"])
+        repo_url = repo_url.strip().replace("ssh://", "https://")
+        repo_url = repo_url.replace("hg://", "https://")
+        cs = repo._run_in_client(["log", "-r", ".", "-T" "{node}"])
+
+    url = pushlog_api_url.format(repo_url, cs)
+    session = requests.Session()
+    try:
+        response = session.get(url)
+    except Exception as e:
+        msg = "Failed to retrieve push timestamp using {}\nError: {}".format(url, e)
+        raise Exception(msg)
+
+    data = response.json()
+
+    date = data['pushdate'][0]
+
+    return datetime.datetime.utcfromtimestamp(date)
+
+
+###
+# Generates timestamp for a locale based on its path.
+# If possible, will use the commit timestamp from HG repository,
+# and if that fails, will generate the timestamp for `now`.
+#
+# The timestamp format is "{year}{month}{day}{hour}{minute}{second}" and
+# the datetime stored in it is using UTC timezone.
+#
+# Args:
+#    path (str) - path to the locale directory
+#
+# Returns:
+#    (str) - a timestamp string
+#
+# Example:
+#    ts = get_timestamp_for_locale("/var/vcs/l10n-central/pl")
+#    ts == "20170914215617"
+###
+def get_timestamp_for_locale(path):
+    dt = None
+    if os.path.isdir(os.path.join(path, '.hg')):
+        dt = get_dt_from_hg(path)
+
+    if dt is None:
+        dt = datetime.datetime.utcnow()
+
+    dt = dt.replace(microsecond=0)
+    return dt.strftime("%Y%m%d%H%M%S")
+
+
 ###
 # Parses multiple defines files into a single key-value pair object.
 #
 # Args:
 #    paths (str) - a comma separated list of paths to defines files
 #
 # Returns:
 #    (dict) - a key-value dict with defines
@@ -233,16 +305,17 @@ def parse_chrome_manifest(path, base_pat
 #
 # Example:
 #    manifest = create_webmanifest(
 #      ['pl'],
 #      '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
 #      '57.0',
 #      '57.0.*',
 #      'Firefox',
+#      '/var/vcs/l10n-central',
 #      {'MOZ_LANG_TITLE': 'Polski'},
 #      chrome_entries
 #    )
 #    manifest == {
 #        'languages': {
 #            'pl': {
 #                'version': '201709121481',
 #                'chrome_resources': {
@@ -272,17 +345,17 @@ def parse_chrome_manifest(path, base_pat
 #            }
 #        },
 #        'version': '57.0',
 #        'name': 'Polski Language Pack',
 #        ...
 #    }
 ###
 def create_webmanifest(locstr, min_app_ver, max_app_ver, app_name,
-                       defines, chrome_entries):
+                       l10n_basedir, defines, chrome_entries):
     locales = map(lambda loc: loc.strip(), locstr.split(','))
     main_locale = locales[0]
 
     author = build_author_string(
         defines['MOZ_LANGPACK_CREATOR'],
         defines['MOZ_LANGPACK_CONTRIBUTORS']
     )
 
@@ -320,33 +393,35 @@ def create_webmanifest(locstr, min_app_v
             else:
                 assert entry['alias'] not in cr
                 cr[entry['alias']] = entry['path']
         else:
             raise Exception('Unknown type {0}'.format(entry['type']))
 
     for loc in locales:
         manifest['languages'][loc] = {
-            'version': min_app_ver,
+            'version': get_timestamp_for_locale(os.path.join(l10n_basedir, loc)),
             'chrome_resources': cr
         }
 
     return json.dumps(manifest, indent=2, ensure_ascii=False, encoding='utf8')
 
 
 def main(args):
     parser = argparse.ArgumentParser()
     parser.add_argument('--locales',
                         help='List of language codes provided by the langpack')
     parser.add_argument('--min-app-ver',
                         help='Min version of the application the langpack is for')
     parser.add_argument('--max-app-ver',
                         help='Max version of the application the langpack is for')
     parser.add_argument('--app-name',
                         help='Name of the application the langpack is for')
+    parser.add_argument('--l10n-basedir',
+                        help='Base directory for locales used in the language pack')
     parser.add_argument('--defines', default=[], nargs='+',
                         help='List of defines files to load data from')
     parser.add_argument('--input',
                         help='Langpack directory.')
 
     args = parser.parse_args(args)
 
     chrome_entries = []
@@ -355,16 +430,17 @@ def main(args):
 
     defines = parse_defines(args.defines)
 
     res = create_webmanifest(
         args.locales,
         args.min_app_ver,
         args.max_app_ver,
         args.app_name,
+        args.l10n_basedir,
         defines,
         chrome_entries
     )
     write_file(os.path.join(args.input, 'manifest.json'), res)
 
 
 if __name__ == '__main__':
     main(sys.argv[1:])
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -220,17 +220,17 @@ langpack-%: libs-%
 	$(call py_action,zip,-C $(DIST)/xpi-stage/locale-$(AB_CD) $(LANGPACK_FILE) install.rdf $(PKG_ZIP_DIRS) chrome.manifest)
 
 langpack-webext-%: LANGPACK_FILE=$(ABS_DIST)/$(PKG_LANGPACK_PATH)$(PKG_LANGPACK_BASENAME).xpi
 langpack-webext-%: AB_CD=$*
 langpack-webext-%: XPI_NAME=locale-$*
 langpack-webext-%: libs-%
 	@echo 'Making new-langpack $(LANGPACK_FILE)'
 	$(NSINSTALL) -D $(DIST)/$(PKG_LANGPACK_PATH)
-	$(call py_action,langpack_manifest,--locales $(AB_CD) --min-app-ver $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name "$(MOZ_APP_DISPLAYNAME)" --defines $(NEW_APP_DEFINES) --input $(DIST)/xpi-stage/locale-$(AB_CD))
+	$(call py_action,langpack_manifest,--locales $(AB_CD) --min-app-ver $(MOZ_APP_VERSION) --max-app-ver $(MOZ_APP_MAXVERSION) --app-name "$(MOZ_APP_DISPLAYNAME)" --l10n-basedir "$(L10NBASEDIR)" --defines $(NEW_APP_DEFINES) --input $(DIST)/xpi-stage/locale-$(AB_CD))
 	$(call py_action,zip,-C $(DIST)/xpi-stage/locale-$(AB_CD) -x **/*.manifest -x **/*.js -x **/*.ini $(LANGPACK_FILE) $(PKG_ZIP_DIRS) manifest.json)
 
 # This variable is to allow the wget-en-US target to know which ftp server to download from
 ifndef EN_US_BINARY_URL 
 EN_US_BINARY_URL = $(error You must set EN_US_BINARY_URL)
 endif
 # In taskcluster the installer comes from another location
 ifndef EN_US_INSTALLER_BINARY_URL