bug 1339706 - l10n-bumper should build platforms from in-tree files. r?rail a=release
We were downloading the json from the dashboard, but the l10n team wants to discontinue that. Instead, we download the revisions from the dashboard for beta and jamun, and use 'default' revisions on central and aurora. We build the platform lists from all-locales and maemo-locales.
I updated the mozilla-beta config to be live, but since we have the mozharness repo and revision pinned in puppet, we will be able to pin to a new revision and point at the jamun config file.
MozReview-Commit-ID: JQcJKMVqcFr
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/jamun.py
@@ -0,0 +1,34 @@
+MULTI_REPO = "projects/jamun"
+config = {
+ "log_name": "l10n_bumper",
+
+ "exes": {
+ # Get around the https warnings
+ "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+ "hgtool.py": ["/usr/local/bin/hgtool.py"],
+ },
+
+ "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+ "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+ "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+ "ssh_key": "~/.ssh/ffxbld_rsa",
+ "ssh_user": "ffxbld",
+
+ "vcs_share_base": "/builds/hg-shared",
+ "version_path": "browser/config/version.txt",
+
+ "bump_configs": [{
+ "path": "mobile/locales/l10n-changesets.json",
+ "format": "json",
+ "name": "Fennec l10n changesets",
+ "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+ "platform_configs": [{
+ "platforms": ["android-api-15", "android"],
+ "path": "mobile/android/locales/all-locales"
+ }, {
+ "platforms": ["android-multilocale"],
+ "path": "mobile/android/locales/maemo-locales"
+ }],
+ }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "releases/mozilla-aurora"
+config = {
+ "log_name": "l10n_bumper",
+
+ "exes": {
+ # Get around the https warnings
+ "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+ "hgtool.py": ["/usr/local/bin/hgtool.py"],
+ },
+
+ "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+ "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+ "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+ "ssh_key": "~/.ssh/ffxbld_rsa",
+ "ssh_user": "ffxbld",
+
+ "vcs_share_base": "/builds/hg-shared",
+ "version_path": "browser/config/version.txt",
+
+ "bump_configs": [{
+ "path": "mobile/locales/l10n-changesets.json",
+ "format": "json",
+ "name": "Fennec l10n changesets",
+ "platform_configs": [{
+ "platforms": ["android-api-15", "android"],
+ "path": "mobile/android/locales/all-locales"
+ }, {
+ "platforms": ["android-multilocale"],
+ "path": "mobile/android/locales/maemo-locales"
+ }],
+ }],
+}
--- a/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
@@ -3,27 +3,32 @@ config = {
"log_name": "l10n_bumper",
"exes": {
# Get around the https warnings
"hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"hgtool.py": ["/usr/local/bin/hgtool.py"],
},
-# "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
-# "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
- "gecko_pull_url": "https://hg.mozilla.org/projects/jamun",
- "gecko_push_url": "ssh://hg.mozilla.org/projects/jamun",
+ "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+ "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
"hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
"ssh_key": "~/.ssh/ffxbld_rsa",
"ssh_user": "ffxbld",
"vcs_share_base": "/builds/hg-shared",
"version_path": "browser/config/version.txt",
"bump_configs": [{
"path": "mobile/locales/l10n-changesets.json",
"format": "json",
"name": "Fennec l10n changesets",
- "url": "https://l10n.mozilla.org/shipping/json-changesets?av=fennec%(MAJOR_VERSION)s&platforms=android&multi_android-multilocale_repo={}&multi_android-multilocale_rev=default&multi_android-multilocale_path=mobile/android/locales/maemo-locales".format(MULTI_REPO),
+ "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+ "platform_configs": [{
+ "platforms": ["android-api-15", "android"],
+ "path": "mobile/android/locales/all-locales"
+ }, {
+ "platforms": ["android-multilocale"],
+ "path": "mobile/android/locales/maemo-locales"
+ }],
}],
}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-central.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "mozilla-central"
+config = {
+ "log_name": "l10n_bumper",
+
+ "exes": {
+ # Get around the https warnings
+ "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+ "hgtool.py": ["/usr/local/bin/hgtool.py"],
+ },
+
+ "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+ "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+ "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+ "ssh_key": "~/.ssh/ffxbld_rsa",
+ "ssh_user": "ffxbld",
+
+ "vcs_share_base": "/builds/hg-shared",
+ "version_path": "browser/config/version.txt",
+
+ "bump_configs": [{
+ "path": "mobile/locales/l10n-changesets.json",
+ "format": "json",
+ "name": "Fennec l10n changesets",
+ "platform_configs": [{
+ "platforms": ["android-api-15", "android"],
+ "path": "mobile/android/locales/all-locales"
+ }, {
+ "platforms": ["android-multilocale"],
+ "path": "mobile/android/locales/maemo-locales"
+ }],
+ }],
+}
--- a/testing/mozharness/scripts/l10n_bumper.py
+++ b/testing/mozharness/scripts/l10n_bumper.py
@@ -24,17 +24,17 @@ try:
assert json
except ImportError:
import json
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.base.errors import HgErrorList
from mozharness.base.vcs.vcsbase import VCSScript
-from mozharness.base.log import ERROR
+from mozharness.base.log import ERROR, FATAL
class L10nBumper(VCSScript):
def __init__(self, require_config_file=True):
super(L10nBumper, self).__init__(
all_actions=[
'clobber',
'check-treestatus',
@@ -66,26 +66,27 @@ class L10nBumper(VCSScript):
os.path.join(
abs_dirs['abs_work_dir'],
self.config.get('gecko_local_dir', os.path.basename(self.config['gecko_pull_url']))
),
})
self.abs_dirs = abs_dirs
return self.abs_dirs
- def hg_commit(self, repo_path, message):
+ def hg_commit(self, path, repo_path, message):
"""
Commits changes in repo_path, with specified user and commit message
"""
user = self.config['hg_user']
hg = self.query_exe('hg', return_type='list')
+ env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
+ cmd = hg + ['add', path]
+ self.run_command(cmd, cwd=repo_path, env=env)
cmd = hg + ['commit', '-u', user, '-m', message]
- env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
status = self.run_command(cmd, cwd=repo_path, env=env)
- return status == 0
def hg_push(self, repo_path):
hg = self.query_exe('hg', return_type='list')
command = hg + ["push", "-e",
"ssh -oIdentityFile=%s -l %s" % (
self.config["ssh_key"], self.config["ssh_user"],
),
self.config["gecko_push_url"]]
@@ -118,20 +119,62 @@ class L10nBumper(VCSScript):
return contents.split('.')
def _build_locale_map(self, old_contents, new_contents):
locale_map = {}
for key in old_contents:
if key not in new_contents:
locale_map[key] = "removed"
for k,v in new_contents.items():
- if old_contents.get(k) != v:
+ if old_contents.get(k, {}).get('revision') != v['revision']:
locale_map[k] = v['revision']
+ elif old_contents.get(k, {}).get('platforms') != v['platforms']:
+ locale_map[k] = v['platforms']
return locale_map
+ def _build_platform_dict(self, bump_config):
+ dirs = self.query_abs_dirs()
+ repo_path = dirs['gecko_local_dir']
+ platform_dict = {}
+ for platform_config in bump_config['platform_configs']:
+ path = os.path.join(repo_path, platform_config['path'])
+ self.info("Reading %s for %s locales..." % (path, platform_config['platforms']))
+ contents = self.read_from_file(path)
+ for locale in contents.splitlines():
+ platforms = platform_dict.get(locale, {}).get('platforms', [])
+ platforms = sorted(list(platform_config['platforms']) + platforms)
+ platform_dict[locale] = {'platforms': platforms}
+ self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
+ return platform_dict
+
+ def _build_revision_dict(self, bump_config, version_list):
+ self.info("Building revision dict...")
+ platform_dict = self._build_platform_dict(bump_config)
+ revision_dict = {}
+ if bump_config.get('revision_url'):
+ repl_dict = {
+ 'MAJOR_VERSION': version_list[0],
+ }
+
+ url = bump_config['revision_url'] % repl_dict
+ path = self.download_file(url, error_level=FATAL)
+ revision_info = self.read_from_file(path)
+ self.info("Got %s" % revision_info)
+ for line in revision_info.splitlines():
+ locale, revision = line.split(' ')
+ if locale in platform_dict:
+ revision_dict[locale] = platform_dict[locale]
+ revision_dict[locale]['revision'] = revision
+ else:
+ for k, v in platform_dict.items():
+ v['revision'] = 'default'
+ revision_dict[k] = v
+ self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
+ return revision_dict
+
def build_commit_message(self, name, locale_map):
revisions = []
comments = ''
for locale, revision in sorted(locale_map.items()):
comments += "%s -> %s\n" % (locale, revision)
message = 'Bumping %s a=l10n-bump\n\n' % (
name,
)
@@ -188,23 +231,18 @@ class L10nBumper(VCSScript):
bump_config['path'])
# For now, assume format == 'json'. When we add desktop support,
# we may need to add flatfile support
if os.path.exists(path):
old_contents = self._read_json(path)
else:
old_contents = {}
- repl_dict = {
- 'MAJOR_VERSION': version_list[0],
- }
+ new_contents = self._build_revision_dict(bump_config, version_list)
- url = bump_config['url'] % repl_dict
- new_contents = self.load_json_url(url)
- self.info("Got %s" % pprint.pformat(new_contents))
if new_contents == old_contents:
continue
# super basic sanity check
if not isinstance(new_contents, dict) or len(new_contents) < 5:
self.error("Cowardly refusing to land a broken-seeming changesets file!")
continue
# Write to disk
@@ -213,17 +251,17 @@ class L10nBumper(VCSScript):
fh.write(content_string + "\n")
fh.close()
locale_map = self._build_locale_map(old_contents, new_contents)
# Commit
message = self.build_commit_message(bump_config['name'],
locale_map)
- self.hg_commit(repo_path, message)
+ self.hg_commit(path, repo_path, message)
changes = True
return changes
def push(self):
dirs = self.query_abs_dirs()
repo_path = dirs['gecko_local_dir']
return self.hg_push(repo_path)