Bug 1262917 - Implement downloadUpdateIgnore404 in release promotion r=jlund a=release DONTBUILD draft
authorRail Aliiev <rail@mozilla.com>
Thu, 07 Apr 2016 15:50:44 -0400
changeset 348639 1893ec89e5bdf8b4e519f4e1e89ae9af9b8daa19
parent 348638 26e409f43a2eb77e70c04376d4145925fbf6896d
child 348777 ddf0d498ec2e79ff33227521ef217b8a4f57c01b
push id14857
push userbmo:rail@mozilla.com
push dateThu, 07 Apr 2016 19:52:40 +0000
reviewersjlund, release
bugs1262917
milestone48.0a1
Bug 1262917 - Implement downloadUpdateIgnore404 in release promotion r=jlund a=release DONTBUILD MozReview-Commit-ID: 8UgJIEwrXTs
release/docker/funsize-update-generator/scripts/funsize.py
--- a/release/docker/funsize-update-generator/scripts/funsize.py
+++ b/release/docker/funsize-update-generator/scripts/funsize.py
@@ -31,19 +31,30 @@ DEFAULT_FILENAME_TEMPLATE = "{appName}-{
 
 def verify_signature(mar, signature):
     log.info("Checking %s signature", mar)
     m = MarFile(mar, signature_versions=[(1, signature)])
     m.verify_signatures()
 
 
 @redo.retriable()
+def downloadable(url):
+    log.debug("Verifying %s", url)
+    r = requests.head(url, timeout=60, allow_redirects=True)
+    if r.status_code == 404:
+        return False
+    else:
+        r.raise_for_status()
+        return True
+
+
+@redo.retriable()
 def download(url, dest, mode=None):
     log.debug("Downloading %s to %s", url, dest)
-    r = requests.get(url)
+    r = requests.get(url, timeout=60)
     r.raise_for_status()
 
     bytes_downloaded = 0
     with open(dest, 'wb') as fd:
         for chunk in r.iter_content(4096):
             fd.write(chunk)
             bytes_downloaded += len(chunk)
 
@@ -187,16 +198,20 @@ def main():
         log.info("Done.")
     except sh.ErrorReturnCode:
         log.warning("Freshclam failed, skipping DB update")
     manifest = []
     for e in task["extra"]["funsize"]["partials"]:
         for mar in (e["from_mar"], e["to_mar"]):
             verify_allowed_url(mar)
 
+        if not downloadable(e["from_mar"]):
+            log.warning("%s cannot be downloaded, skipping...", e["from_mar"])
+            continue
+
         work_env = WorkEnv()
         # TODO: run setup once
         work_env.setup()
         complete_mars = {}
         for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
             dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
             unpack_dir = os.path.join(work_env.workdir, mar_type)
             download(f, dest)