bug 1339706 - l10n-bumper should build platforms from in-tree files. r?rail a=release draft
authorAki Sasaki <asasaki@mozilla.com>
Thu, 16 Feb 2017 14:17:03 -0800
changeset 486238 27adeaa8e0e987f0371bb79fcd77d80cc94b06ba
parent 485605 05d7ba5c91058088301184684dc5a582a6078ca9
child 486239 048ad72f1826b426f5a7519f1ec81654009003a0
push id45922
push userasasaki@mozilla.com
push dateFri, 17 Feb 2017 17:23:02 +0000
reviewersrail, release
bugs1339706
milestone54.0a1
bug 1339706 - l10n-bumper should build platforms from in-tree files. r?rail a=release We were downloading the json from the dashboard, but the l10n team wants to discontinue that. Instead, we download the revisions from the dashboard for beta and jamun, and use 'default' revisions on central and aurora. We build the platform lists from all-locales and maemo-locales. I updated the mozilla-beta config to be live, but since we have the mozharness repo and revision pinned in puppet, we will be able to pin to a new revision and point at the jamun config file. MozReview-Commit-ID: JQcJKMVqcFr
testing/mozharness/configs/l10n_bumper/jamun.py
testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
testing/mozharness/configs/l10n_bumper/mozilla-beta.py
testing/mozharness/configs/l10n_bumper/mozilla-central.py
testing/mozharness/scripts/l10n_bumper.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/jamun.py
@@ -0,0 +1,34 @@
+MULTI_REPO = "projects/jamun"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "releases/mozilla-aurora"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
--- a/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
@@ -3,27 +3,32 @@ config = {
     "log_name": "l10n_bumper",
 
     "exes": {
         # Get around the https warnings
         "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
         "hgtool.py": ["/usr/local/bin/hgtool.py"],
     },
 
-#    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
-#    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
-    "gecko_pull_url": "https://hg.mozilla.org/projects/jamun",
-    "gecko_push_url": "ssh://hg.mozilla.org/projects/jamun",
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
 
     "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
     "ssh_key": "~/.ssh/ffxbld_rsa",
     "ssh_user": "ffxbld",
 
     "vcs_share_base": "/builds/hg-shared",
     "version_path": "browser/config/version.txt",
 
     "bump_configs": [{
         "path": "mobile/locales/l10n-changesets.json",
         "format": "json",
         "name": "Fennec l10n changesets",
-        "url": "https://l10n.mozilla.org/shipping/json-changesets?av=fennec%(MAJOR_VERSION)s&platforms=android&multi_android-multilocale_repo={}&multi_android-multilocale_rev=default&multi_android-multilocale_path=mobile/android/locales/maemo-locales".format(MULTI_REPO),
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
     }],
 }
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-central.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "mozilla-central"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
--- a/testing/mozharness/scripts/l10n_bumper.py
+++ b/testing/mozharness/scripts/l10n_bumper.py
@@ -24,17 +24,17 @@ try:
     assert json
 except ImportError:
     import json
 
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.errors import HgErrorList
 from mozharness.base.vcs.vcsbase import VCSScript
-from mozharness.base.log import ERROR
+from mozharness.base.log import ERROR, FATAL
 
 
 class L10nBumper(VCSScript):
     def __init__(self, require_config_file=True):
         super(L10nBumper, self).__init__(
             all_actions=[
                 'clobber',
                 'check-treestatus',
@@ -66,26 +66,27 @@ class L10nBumper(VCSScript):
             os.path.join(
                 abs_dirs['abs_work_dir'],
                 self.config.get('gecko_local_dir', os.path.basename(self.config['gecko_pull_url']))
             ),
         })
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
-    def hg_commit(self, repo_path, message):
+    def hg_commit(self, path, repo_path, message):
         """
         Commits changes in repo_path, with specified user and commit message
         """
         user = self.config['hg_user']
         hg = self.query_exe('hg', return_type='list')
+        env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
+        cmd = hg + ['add', path]
+        self.run_command(cmd, cwd=repo_path, env=env)
         cmd = hg + ['commit', '-u', user, '-m', message]
-        env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
         status = self.run_command(cmd, cwd=repo_path, env=env)
-        return status == 0
 
     def hg_push(self, repo_path):
         hg = self.query_exe('hg', return_type='list')
         command = hg + ["push", "-e",
                         "ssh -oIdentityFile=%s -l %s" % (
                             self.config["ssh_key"], self.config["ssh_user"],
                         ),
                         self.config["gecko_push_url"]]
@@ -118,20 +119,62 @@ class L10nBumper(VCSScript):
         return contents.split('.')
 
     def _build_locale_map(self, old_contents, new_contents):
         locale_map = {}
         for key in old_contents:
             if key not in new_contents:
                 locale_map[key] = "removed"
         for k,v in new_contents.items():
-            if old_contents.get(k) != v:
+            if old_contents.get(k, {}).get('revision') != v['revision']:
                 locale_map[k] = v['revision']
+            elif old_contents.get(k, {}).get('platforms') != v['platforms']:
+                locale_map[k] = v['platforms']
         return locale_map
 
+    def _build_platform_dict(self, bump_config):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs['gecko_local_dir']
+        platform_dict = {}
+        for platform_config in bump_config['platform_configs']:
+            path = os.path.join(repo_path, platform_config['path'])
+            self.info("Reading %s for %s locales..." % (path, platform_config['platforms']))
+            contents = self.read_from_file(path)
+            for locale in contents.splitlines():
+                platforms = platform_dict.get(locale, {}).get('platforms', [])
+                platforms = sorted(list(platform_config['platforms']) + platforms)
+                platform_dict[locale] = {'platforms': platforms}
+        self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
+        return platform_dict
+
+    def _build_revision_dict(self, bump_config, version_list):
+        self.info("Building revision dict...")
+        platform_dict = self._build_platform_dict(bump_config)
+        revision_dict = {}
+        if bump_config.get('revision_url'):
+            repl_dict = {
+                'MAJOR_VERSION': version_list[0],
+            }
+
+            url = bump_config['revision_url'] % repl_dict
+            path = self.download_file(url, error_level=FATAL)
+            revision_info = self.read_from_file(path)
+            self.info("Got %s" % revision_info)
+            for line in revision_info.splitlines():
+                locale, revision = line.split(' ')
+                if locale in platform_dict:
+                    revision_dict[locale] = platform_dict[locale]
+                    revision_dict[locale]['revision'] = revision
+        else:
+            for k, v in platform_dict.items():
+                v['revision'] = 'default'
+                revision_dict[k] = v
+        self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
+        return revision_dict
+
     def build_commit_message(self, name, locale_map):
         revisions = []
         comments = ''
         for locale, revision in sorted(locale_map.items()):
             comments += "%s -> %s\n" % (locale, revision)
         message = 'Bumping %s a=l10n-bump\n\n' % (
             name,
         )
@@ -188,23 +231,18 @@ class L10nBumper(VCSScript):
                                 bump_config['path'])
             # For now, assume format == 'json'.  When we add desktop support,
             # we may need to add flatfile support
             if os.path.exists(path):
                 old_contents = self._read_json(path)
             else:
                 old_contents = {}
 
-            repl_dict = {
-                'MAJOR_VERSION': version_list[0],
-            }
+            new_contents = self._build_revision_dict(bump_config, version_list)
 
-            url = bump_config['url'] % repl_dict
-            new_contents = self.load_json_url(url)
-            self.info("Got %s" % pprint.pformat(new_contents))
             if new_contents == old_contents:
                 continue
             # super basic sanity check
             if not isinstance(new_contents, dict) or len(new_contents) < 5:
                 self.error("Cowardly refusing to land a broken-seeming changesets file!")
                 continue
 
             # Write to disk
@@ -213,17 +251,17 @@ class L10nBumper(VCSScript):
             fh.write(content_string + "\n")
             fh.close()
 
             locale_map = self._build_locale_map(old_contents, new_contents)
 
             # Commit
             message = self.build_commit_message(bump_config['name'],
                                                 locale_map)
-            self.hg_commit(repo_path, message)
+            self.hg_commit(path, repo_path, message)
             changes = True
         return changes
 
     def push(self):
         dirs = self.query_abs_dirs()
         repo_path = dirs['gecko_local_dir']
         return self.hg_push(repo_path)