--- a/taskcluster/docker/funsize-balrog-submitter/Dockerfile
+++ b/taskcluster/docker/funsize-balrog-submitter/Dockerfile
@@ -21,14 +21,15 @@ RUN hg clone https://hg.mozilla.org/buil
RUN useradd -d /home/worker -s /bin/bash -m worker
RUN mkdir /home/worker/bin
COPY scripts/* /home/worker/bin/
RUN mkdir /home/worker/keys
COPY *.pubkey /home/worker/keys/
COPY runme.sh /runme.sh
-RUN chmod 755 /home/worker/bin/* /runme.sh
+COPY submit_complete.sh /submit_complete.sh
+RUN chmod 755 /home/worker/bin/* /runme.sh /submit_complete.sh
ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter-complete.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function
+
+import site
+import os
+import logging
+import argparse
+import json
+
+site.addsitedir("/home/worker/tools/lib/python")
+
+from balrog.submitter.cli import ReleaseSubmitterV4
+from util.retry import retry
+
+log = logging.getLogger(__name__)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--manifest", required=True)
+ parser.add_argument("-a", "--api-root", required=True,
+ help="Balrog API root")
+ parser.add_argument("-v", "--verbose", action="store_const",
+ dest="loglevel", const=logging.DEBUG,
+ default=logging.INFO)
+ parser.add_argument("--product", help="Override product name from application.ini")
+ args = parser.parse_args()
+ logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
+ level=args.loglevel)
+ logging.getLogger("requests").setLevel(logging.WARNING)
+ logging.getLogger("boto").setLevel(logging.WARNING)
+
+ balrog_username = os.environ.get("BALROG_USERNAME")
+ balrog_password = os.environ.get("BALROG_PASSWORD")
+ suffix = os.environ.get("BALROG_BLOB_SUFFIX")
+ if not balrog_username and not balrog_password:
+ raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
+ "variables should be set")
+ if not suffix:
+ raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set")
+
+ manifest = json.load(open(args.manifest))
+ auth = (balrog_username, balrog_password)
+
+ for e in manifest:
+ complete_info = [{
+ "hash": e["hash"],
+ "size": e["size"],
+ }]
+
+ submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
+ suffix=suffix)
+ productName = args.product or e["appName"]
+ retry(lambda: submitter.run(
+ platform=e["platform"], productName=productName,
+ version=e["toVersion"],
+ build_number=e["toBuildNumber"],
+ appVersion=e["version"], extVersion=e["version"],
+ buildID=e["to_buildid"], locale=e["locale"],
+ hashFunction='sha512', completeInfo=complete_info),
+ attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
+ )
+
+
+if __name__ == '__main__':
+ main()
--- a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
+++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
@@ -128,16 +128,18 @@ def main():
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
+ # blob suffix used for releases only
+ suffix = os.environ.get("BALROG_BLOB_SUFFIX")
s3_bucket = os.environ.get("S3_BUCKET")
aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
if not (s3_bucket and aws_access_key_id and aws_secret_access_key):
log.warn("Skipping S3 uploads...")
uploads_enabled = False
else:
@@ -164,18 +166,21 @@ def main():
"size": e["size"],
}]
if "previousVersion" in e and "previousBuildNumber" in e:
log.info("Release style balrog submission")
partial_info[0]["previousVersion"] = e["previousVersion"]
partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
- dummy=args.dummy)
+ dummy=args.dummy, suffix=suffix)
productName = args.product or e["appName"]
+ if suffix:
+ log.warning("Not submitting complete info")
+ complete_info = None
retry(lambda: submitter.run(
platform=e["platform"], productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"], extVersion=e["version"],
buildID=e["to_buildid"], locale=e["locale"],
hashFunction='sha512',
partialInfo=partial_info, completeInfo=complete_info),
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/funsize-balrog-submitter/submit_complete.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+test $PARENT_TASK_ARTIFACTS_URL_PREFIX
+test $BALROG_API_ROOT
+# BALROG_BLOB_SUFFIX is used by the script implicitly to avoid possible CLI
+# issues with suffixes starting with "-"
+test $BALROG_BLOB_SUFFIX
+
+
+ARTIFACTS_DIR="/home/worker/artifacts"
+mkdir -p "$ARTIFACTS_DIR"
+
+curl --location --retry 10 --retry-delay 10 -o "$ARTIFACTS_DIR/manifest.json" \
+ "$PARENT_TASK_ARTIFACTS_URL_PREFIX/manifest.json"
+
+cat "$ARTIFACTS_DIR/manifest.json"
+python /home/worker/bin/funsize-balrog-submitter-complete.py \
+ --manifest "$ARTIFACTS_DIR/manifest.json" \
+ -a "$BALROG_API_ROOT" \
+ --verbose \
+ $EXTRA_BALROG_SUBMITTER_PARAMS
--- a/taskcluster/docker/funsize-update-generator/scripts/recompress.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/recompress.py
@@ -1,11 +1,12 @@
#!/usr/bin/env python
from __future__ import absolute_import, print_function
+import ConfigParser
import argparse
import functools
import hashlib
import json
import logging
import os
import shutil
import tempfile
@@ -24,16 +25,36 @@ ALLOWED_URL_PREFIXES = [
"https://mozilla-nightly-updates.s3.amazonaws.com",
"https://queue.taskcluster.net/",
"http://ftp.mozilla.org/",
"http://download.mozilla.org/",
"https://archive.mozilla.org/",
]
+def find_file(directory, filename):
+ log.debug("Searching for %s in %s", filename, directory)
+ for root, dirs, files in os.walk(directory):
+ if filename in files:
+ f = os.path.join(root, filename)
+ log.debug("Found %s", f)
+ return f
+
+
+def get_option(directory, filename, section, option):
+ log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory,
+ filename)
+ f = find_file(directory, filename)
+ config = ConfigParser.ConfigParser()
+ config.read(f)
+ rv = config.get(section, option)
+ log.debug("Found %s", rv)
+ return rv
+
+
def verify_signature(mar, certs):
log.info("Checking %s signature", mar)
with open(mar, 'rb') as mar_fh:
m = MarReader(mar_fh)
m.verify(verify_key=certs.get(m.signature_type))
def is_lzma_compressed_mar(mar):
@@ -190,16 +211,26 @@ def main():
unpack(work_env, dest, unpack_dir)
log.info("AV-scanning %s ...", unpack_dir)
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
mar_data = {
"file_to_sign": output_filename,
"hash": get_hash(dest),
+ "appName": get_option(unpack_dir, filename="application.ini",
+ section="App", option="Name"),
+ "version": get_option(unpack_dir, filename="application.ini",
+ section="App", option="Version"),
+ "to_buildid": get_option(unpack_dir, filename="application.ini",
+ section="App", option="BuildID"),
+ "toVersion": e["toVersion"],
+ "toBuildNumber": e["toBuildNumber"],
+ "platform": e["platform"],
+ "locale": e["locale"],
}
shutil.copy(dest, os.path.join(args.artifacts_dir, output_filename))
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
--- a/testing/mozharness/configs/releases/dev_updates_firefox_beta.py
+++ b/testing/mozharness/configs/releases/dev_updates_firefox_beta.py
@@ -6,34 +6,39 @@ config = {
"repo": "https://hg.mozilla.org/users/raliiev_mozilla.com/tools",
"branch": "default",
"dest": "tools",
"vcs": "hg",
},
"vcs_share_base": "/builds/hg-shared",
# TODO: use real repo
"push_dest": "ssh://hg.mozilla.org/users/raliiev_mozilla.com/tools",
- # date repo used for staging beta
- "shipped-locales-url": "https://hg.mozilla.org/projects/date/raw-file/{revision}/browser/locales/shipped-locales",
+ # jamun repo used for staging beta
+ "shipped-locales-url": "https://hg.mozilla.org/projects/jamun/raw-file/{revision}/browser/locales/shipped-locales",
"ignore_no_changes": True,
"ssh_user": "ffxbld",
"ssh_key": "~/.ssh/ffxbld_rsa",
"archive_domain": "ftp.stage.mozaws.net",
"archive_prefix": "https://ftp.stage.mozaws.net/pub",
"previous_archive_prefix": "https://archive.mozilla.org/pub",
"download_domain": "download.mozilla.org",
- "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
+ "balrog_url": "http://54.90.211.22:9090",
"balrog_username": "balrog-stage-ffxbld",
"update_channels": {
"beta-dev": {
"version_regex": r"^(\d+\.\d+(b\d+)?)$",
"requires_mirrors": True,
- # TODO - when we use a real repo, rename this file # s/MozDate/MozBeta-dev/
- "patcher_config": "mozDate-branch-patcher2.cfg",
+ # TODO - when we use a real repo, rename this file # s/MozJamun/MozBeta-dev/
+ "patcher_config": "mozJamun-branch-patcher2.cfg",
"update_verify_channel": "beta-dev-localtest",
"mar_channel_ids": [],
"channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"],
"rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
"publish_rules": ["firefox-beta"],
+ "bz2_blob_suffix": "-bz2",
+ "bz2_rules_to_update": ["firefox-beta-cdntest-bz2", "firefox-beta-localtest-bz2"],
+ "bz2_publish_rules": [652],
+ "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+ "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
}
},
"balrog_use_dummy_suffix": False,
}
--- a/testing/mozharness/configs/releases/dev_updates_firefox_release.py
+++ b/testing/mozharness/configs/releases/dev_updates_firefox_release.py
@@ -15,37 +15,47 @@ config = {
"shipped-locales-url": "https://hg.mozilla.org/projects/jamun/raw-file/{revision}/browser/locales/shipped-locales",
"ignore_no_changes": True,
"ssh_user": "ffxbld",
"ssh_key": "~/.ssh/ffxbld_rsa",
"archive_domain": "ftp.stage.mozaws.net",
"archive_prefix": "https://ftp.stage.mozaws.net/pub",
"previous_archive_prefix": "https://archive.mozilla.org/pub",
"download_domain": "download.mozilla.org",
- "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
+ "balrog_url": "http://54.90.211.22:9090",
"balrog_username": "balrog-stage-ffxbld",
"update_channels": {
"beta-dev": {
"version_regex": r"^(\d+\.\d+(b\d+)?)$",
"requires_mirrors": False,
"patcher_config": "mozDate-branch-patcher2.cfg",
"update_verify_channel": "beta-dev-localtest",
"mar_channel_ids": [
"firefox-mozilla-beta-dev", "firefox-mozilla-release-dev",
],
"channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"],
"rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
"publish_rules": ["firefox-beta"],
"schedule_asap": True,
+ "bz2_blob_suffix": "-bz2",
+ "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+ "bz2_publish_rules": [624],
+ "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+ "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
},
"release-dev": {
"version_regex": r"^\d+\.\d+(\.\d+)?$",
"requires_mirrors": True,
"patcher_config": "mozJamun-branch-patcher2.cfg",
"update_verify_channel": "release-dev-localtest",
"mar_channel_ids": [],
"channel_names": ["release-dev", "release-dev-localtest", "release-dev-cdntest"],
"rules_to_update": ["firefox-release-dev-cdntest", "firefox-release-dev-localtest"],
"publish_rules": ["firefox-release"],
+ "bz2_blob_suffix": "-bz2",
+ "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+ "bz2_publish_rules": [624],
+ "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+ "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
},
},
"balrog_use_dummy_suffix": False,
}
--- a/testing/mozharness/configs/releases/updates_firefox_release.py
+++ b/testing/mozharness/configs/releases/updates_firefox_release.py
@@ -37,12 +37,17 @@ config = {
"version_regex": r"^\d+\.\d+(\.\d+)?$",
"requires_mirrors": True,
"patcher_config": "mozRelease-branch-patcher2.cfg",
"update_verify_channel": "release-localtest",
"mar_channel_ids": [],
"channel_names": ["release", "release-localtest", "release-cdntest"],
"rules_to_update": ["firefox-release-cdntest", "firefox-release-localtest"],
"publish_rules": [145],
+ "bz2_blob_suffix": "-bz2",
+ "bz2_rules_to_update": ["firefox-release-cdntest-bz2", "firefox-release-localtest-bz2"],
+ "bz2_publish_rules": [624],
+ "complete_mar_filename_pattern": '%s-%s.bz2.complete.mar',
+ "complete_mar_bouncer_product_pattern": '%s-%s-complete-bz2',
},
},
"balrog_use_dummy_suffix": False,
}
--- a/testing/mozharness/scripts/release/publish_balrog.py
+++ b/testing/mozharness/scripts/release/publish_balrog.py
@@ -50,17 +50,18 @@ class PublishBalrog(MercurialScript, Bui
# taskcluster properties
self.read_buildbot_config()
if not self.buildbot_config:
self.warning("Skipping buildbot properties overrides")
return
# TODO: version and appVersion should come from repo
props = self.buildbot_config["properties"]
for prop in ['product', 'version', 'build_number', 'channels',
- 'balrog_api_root', 'schedule_at', 'background_rate']:
+ 'balrog_api_root', 'schedule_at', 'background_rate',
+ 'publish_bz2_blob']:
if props.get(prop):
self.info("Overriding %s with %s" % (prop, props[prop]))
self.config[prop] = props.get(prop)
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
self.abs_dirs = super(PublishBalrog, self).query_abs_dirs()
@@ -84,16 +85,20 @@ class PublishBalrog(MercurialScript, Bui
def pull(self):
super(PublishBalrog, self).pull(
repos=self.query_repos())
def submit_to_balrog(self):
for _, channel_config in self.query_channel_configs():
self._submit_to_balrog(channel_config)
+ if 'publish_bz2_blob' in self.config and \
+ self.config['publish_bz2_blob']:
+ for _, channel_config in self.query_channel_configs():
+ self._submit_to_balrog_bz2(channel_config)
def _submit_to_balrog(self, channel_config):
dirs = self.query_abs_dirs()
auth = os.path.join(os.getcwd(), self.config['credentials_file'])
cmd = [
sys.executable,
os.path.join(dirs["abs_tools_dir"],
"scripts/build-promotion/balrog-release-shipper.py")]
@@ -118,12 +123,50 @@ class PublishBalrog(MercurialScript, Bui
elif self.config.get("schedule_at"):
cmd.extend(["--schedule-at", self.config["schedule_at"]])
if self.config.get("background_rate"):
cmd.extend(["--background-rate", str(self.config["background_rate"])])
self.retry(lambda: self.run_command(cmd, halt_on_failure=True),
error_level=FATAL)
+ def _submit_to_balrog_bz2(self, channel_config):
+ dirs = self.query_abs_dirs()
+ # Use env varialbe instead of command line to avoid issues with blob
+ # names starting with "-", e.g. "-bz2"
+ env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]}
+ auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+ cmd = [
+ sys.executable,
+ os.path.join(dirs["abs_tools_dir"],
+ "scripts/build-promotion/balrog-release-shipper.py")]
+ cmd.extend([
+ "--api-root", self.config["balrog_api_root"],
+ "--credentials-file", auth,
+ "--username", self.config["balrog_username"],
+ "--version", self.config["version"],
+ "--product", self.config["product"],
+ "--build-number", str(self.config["build_number"]),
+ "--suffix", channel_config["bz2_blob_suffix"],
+ "--verbose",
+ ])
+ for r in channel_config["bz2_publish_rules"]:
+ cmd.extend(["--rules", str(r)])
+ if channel_config.get("schedule_asap"):
+ # RC releases going to the beta channel have no ETA set for the
+ # RC-to-beta push. The corresponding task is scheduled after we
+ # resolve the push-to-beta human decision task, so we can schedule
+ # it ASAP plus some additional 30m to avoid retry() to fail.
+ schedule_at = datetime.utcnow() + timedelta(minutes=30)
+ cmd.extend(["--schedule-at", schedule_at.isoformat()])
+ elif self.config.get("schedule_at"):
+ cmd.extend(["--schedule-at", self.config["schedule_at"]])
+ if self.config.get("background_rate"):
+ cmd.extend(["--background-rate", str(self.config["background_rate"])])
+
+ self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env),
+ error_level=FATAL)
+
+
# __main__ {{{1
if __name__ == '__main__':
PublishBalrog().run_and_exit()
--- a/testing/mozharness/scripts/release/updates.py
+++ b/testing/mozharness/scripts/release/updates.py
@@ -89,17 +89,18 @@ class UpdatesBumper(MercurialScript, Bui
# taskcluster properties
self.read_buildbot_config()
if not self.buildbot_config:
self.warning("Skipping buildbot properties overrides")
return
# TODO: version and appVersion should come from repo
props = self.buildbot_config["properties"]
for prop in ['product', 'version', 'build_number', 'revision',
- 'appVersion', 'balrog_api_root', "channels"]:
+ 'appVersion', 'balrog_api_root', "channels",
+ 'generate_bz2_blob']:
if props.get(prop):
self.info("Overriding %s with %s" % (prop, props[prop]))
self.config[prop] = props.get(prop)
partials = [v.strip() for v in props["partial_versions"].split(",")]
self.config["partial_versions"] = [v.split("build") for v in partials]
self.config["platforms"] = [p.strip() for p in
props["platforms"].split(",")]
@@ -264,16 +265,20 @@ class UpdatesBumper(MercurialScript, Bui
build_number=self.config["build_number"])
for t in tags]
self.hg_tag(cwd=dirs["abs_tools_dir"], tags=tags,
user=self.config["hg_user"], force=True)
def submit_to_balrog(self):
for _, channel_config in self.query_channel_configs():
self._submit_to_balrog(channel_config)
+ if 'generate_bz2_blob' in self.config and \
+ self.config['generate_bz2_blob']:
+ for _, channel_config in self.query_channel_configs():
+ self._submit_to_balrog_bz2(channel_config)
def _submit_to_balrog(self, channel_config):
dirs = self.query_abs_dirs()
auth = os.path.join(os.getcwd(), self.config['credentials_file'])
cmd = [
sys.executable,
os.path.join(dirs["abs_tools_dir"],
"scripts/build-promotion/balrog-release-pusher.py")]
@@ -301,11 +306,65 @@ class UpdatesBumper(MercurialScript, Bui
cmd.extend(["--partial-update", partial])
if channel_config["requires_mirrors"]:
cmd.append("--requires-mirrors")
if self.config["balrog_use_dummy_suffix"]:
cmd.append("--dummy")
self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
+ def _submit_to_balrog_bz2(self, channel_config):
+ if "bz2_blob_suffix" not in channel_config:
+ self.info("No need to generate BZ2 blob")
+ return
+
+ dirs = self.query_abs_dirs()
+ # Use env varialbe instead of command line to avoid issues with blob
+ # names starting with "-", e.g. "-bz2"
+ env = {"BALROG_BLOB_SUFFIX": channel_config["bz2_blob_suffix"]}
+ auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+ cmd = [
+ sys.executable,
+ os.path.join(dirs["abs_tools_dir"],
+ "scripts/build-promotion/balrog-release-pusher.py")]
+ cmd.extend([
+ "--api-root", self.config["balrog_api_root"],
+ "--download-domain", self.config["download_domain"],
+ "--archive-domain", self.config["archive_domain"],
+ "--credentials-file", auth,
+ "--product", self.config["product"],
+ "--version", self.config["version"],
+ "--build-number", str(self.config["build_number"]),
+ "--app-version", self.config["appVersion"],
+ "--username", self.config["balrog_username"],
+ "--complete-mar-filename-pattern",
+ channel_config["complete_mar_filename_pattern"],
+ "--complete-mar-bouncer-product-pattern",
+ channel_config["complete_mar_bouncer_product_pattern"],
+ "--verbose",
+ ])
+
+ for v, build_number in self.query_matching_partials(channel_config):
+ if v < "56.0":
+ self.info("Adding %s to partials" % v)
+ partial = "{version}build{build_number}".format(
+ version=v, build_number=build_number)
+ cmd.extend(["--partial-update", partial])
+ else:
+ self.info("Not adding %s to partials" % v)
+
+ for c in channel_config["channel_names"]:
+ cmd.extend(["--channel", c])
+ for r in channel_config["bz2_rules_to_update"]:
+ cmd.extend(["--rule-to-update", r])
+ for p in self.config["platforms"]:
+ cmd.extend(["--platform", p])
+ if channel_config["requires_mirrors"]:
+ cmd.append("--requires-mirrors")
+ if self.config["balrog_use_dummy_suffix"]:
+ cmd.append("--dummy")
+
+ self.retry(lambda: self.run_command(cmd, halt_on_failure=True, env=env))
+
+
# __main__ {{{1
if __name__ == '__main__':
UpdatesBumper().run_and_exit()