Bug 1262917 - Implement downloadUpdateIgnore404 in release promotion r=jlund a=release DONTBUILD
MozReview-Commit-ID: 8UgJIEwrXTs
--- a/release/docker/funsize-update-generator/scripts/funsize.py
+++ b/release/docker/funsize-update-generator/scripts/funsize.py
@@ -31,19 +31,30 @@ DEFAULT_FILENAME_TEMPLATE = "{appName}-{
def verify_signature(mar, signature):
log.info("Checking %s signature", mar)
m = MarFile(mar, signature_versions=[(1, signature)])
m.verify_signatures()
@redo.retriable()
+def downloadable(url):
+ log.debug("Verifying %s", url)
+ r = requests.head(url, timeout=60, allow_redirects=True)
+ if r.status_code == 404:
+ return False
+ else:
+ r.raise_for_status()
+ return True
+
+
+@redo.retriable()
def download(url, dest, mode=None):
log.debug("Downloading %s to %s", url, dest)
- r = requests.get(url)
+ r = requests.get(url, timeout=60)
r.raise_for_status()
bytes_downloaded = 0
with open(dest, 'wb') as fd:
for chunk in r.iter_content(4096):
fd.write(chunk)
bytes_downloaded += len(chunk)
@@ -187,16 +198,20 @@ def main():
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
+ if not downloadable(e["from_mar"]):
+ log.warning("%s cannot be downloaded, skipping...", e["from_mar"])
+ continue
+
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
download(f, dest)