bug 1385946 Final update to funsize docker images r=catlee draft
authorSimon Fraser <sfraser@mozilla.com>
Tue, 01 Aug 2017 22:23:50 +0100
changeset 619307 861b790a36c90b467141413f9f93815f89ac7f4d
parent 619065 ef9a0f01e4f68214f0ff8f4631783b8a0e075a82
child 640363 c8a89eef25916b6a2726c7f5cff1d68d56134e65
push id71643
push usersfraser@mozilla.com
push dateTue, 01 Aug 2017 21:24:22 +0000
reviewerscatlee
bugs1385946
milestone56.0a1
bug 1385946 Final update to funsize docker images r=catlee MozReview-Commit-ID: 39geOuOeco3
taskcluster/docker/funsize-balrog-submitter/Dockerfile
taskcluster/docker/funsize-balrog-submitter/requirements.txt
taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
taskcluster/docker/funsize-update-generator/Dockerfile
taskcluster/docker/funsize-update-generator/README
taskcluster/docker/funsize-update-generator/requirements.txt
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-balrog-submitter/Dockerfile
+++ b/taskcluster/docker/funsize-balrog-submitter/Dockerfile
@@ -2,17 +2,18 @@ FROM ubuntu:vivid
 MAINTAINER Rail Aliiev <rail@mozilla.com>
 
 # Required software
 ENV DEBIAN_FRONTEND noninteractive
 # Chain apt-get commands with apt-get clean in a single docker RUN
 # to make sure that files are removed within a single docker layer
 RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
-    python mercurial curl python-boto python-setuptools python-cryptography && \
+    python mercurial curl python-boto python-setuptools python-cryptography \
+    python-dev gcc liblzma-dev && \
     apt-get clean
 
 COPY requirements.txt /tmp/
 # python-pip installs a lot of dependencies increasing the size of an image
 # drastically.
 RUN easy_install pip
 RUN pip install -r /tmp/requirements.txt
 
--- a/taskcluster/docker/funsize-balrog-submitter/requirements.txt
+++ b/taskcluster/docker/funsize-balrog-submitter/requirements.txt
@@ -1,1 +1,1 @@
-mar==2.1.1
+mar==2.1.2
--- a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
+++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
@@ -47,17 +47,17 @@ def download(url, dest, mode=None):
         log.debug("chmod %o %s", mode, dest)
         os.chmod(dest, mode)
 
 
 def verify_signature(mar, certs):
     log.info("Checking %s signature", mar)
     with open(mar, 'rb') as mar_fh:
         m = MarReader(mar_fh)
-        m.verify(verify_key=certs.get(m.signature_type()))
+        m.verify(verify_key=certs.get(m.signature_type))
 
 
 def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key,
                       mar_url, mar_dest, signing_certs):
     conn = S3Connection(aws_access_key_id, aws_secret_access_key)
     bucket = conn.get_bucket(bucket_name)
     _, dest = tempfile.mkstemp()
     log.info("Downloading %s to %s...", mar_url, dest)
@@ -140,22 +140,22 @@ def main():
         uploads_enabled = False
     else:
         uploads_enabled = True
 
     manifest = json.load(open(args.manifest))
     auth = (balrog_username, balrog_password)
 
     signing_certs = {
-        'sha1': args.sha1_signing_cert,
-        'sha384': args.sha384_signing_cert,
+        'sha1': open(args.sha1_signing_cert, 'rb').read(),
+        'sha384': open(args.sha384_signing_cert, 'rb').read(),
     }
 
-    assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048)
-    assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096)
+    assert(get_keysize(signing_certs['sha1']) == 2048)
+    assert(get_keysize(signing_certs['sha384']) == 4096)
 
     for e in manifest:
         complete_info = [{
             "hash": e["to_hash"],
             "size": e["to_size"],
         }]
         partial_info = [{
             "hash": e["hash"],
--- a/taskcluster/docker/funsize-update-generator/Dockerfile
+++ b/taskcluster/docker/funsize-update-generator/Dockerfile
@@ -3,31 +3,34 @@ MAINTAINER Rail Aliiev <rail@mozilla.com
 
 # Required software
 ENV DEBIAN_FRONTEND noninteractive
 # Chain apt-get commands with apt-get clean in a single docker RUN
 # to make sure that files are removed within a single docker layer
 RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
     python python-setuptools python-cryptography  libgetopt-simple-perl \
-    bzip2 clamav clamav-freshclam python-requests python-sh curl && \
+    bzip2 clamav clamav-freshclam python-requests python-sh curl \
+    python-dev gcc liblzma-dev xz-utils && \
     apt-get clean
 RUN useradd -d /home/worker -s /bin/bash -m worker
 COPY requirements.txt /tmp/
+
+# Freshclam may be flaky, retry if it fails
+RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done
+
 # python-pip installs a lot of dependencies increasing the size of an image
 # drastically. Using easy_install saves us almost 200M.
 RUN easy_install pip
 RUN pip install -r /tmp/requirements.txt
 
 # scripts
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
 COPY runme.sh /runme.sh
 RUN chmod 755 /home/worker/bin/* /runme.sh
 RUN mkdir /home/worker/keys
 COPY *.pubkey /home/worker/keys/
-# Freshclam may be flaky, retry if it fails
-RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done
 
 ENV           HOME          /home/worker
 ENV           SHELL         /bin/bash
 ENV           USER          worker
 ENV           LOGNAME       worker
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/funsize-update-generator/README
@@ -0,0 +1,7 @@
+
+To run this locally for testing/development purposes:
+
+1. Find a funsize generating task ID
+2. make pull DOCKERIO_USERNAME=mozillareleases
+3. docker run -t -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID=LD5HUGP5QNeQdFKNTTuyCg mozillareleases/funsize-update-generator /runme.sh
+
--- a/taskcluster/docker/funsize-update-generator/requirements.txt
+++ b/taskcluster/docker/funsize-update-generator/requirements.txt
@@ -1,2 +1,2 @@
-mar==2.1.1
+mar==2.1.2
 redo
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -31,22 +31,27 @@ ALLOWED_URL_PREFIXES = [
 DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
                             "{locale}-{from_buildid}-{to_buildid}.partial.mar"
 
 
 def verify_signature(mar, certs):
     log.info("Checking %s signature", mar)
     with open(mar, 'rb') as mar_fh:
         m = MarReader(mar_fh)
-        m.verify(verify_key=certs.get(m.signature_type()))
+        m.verify(verify_key=certs.get(m.signature_type))
 
 
 def is_lzma_compressed_mar(mar):
     log.info("Checking %s for lzma compression", mar)
-    return MarReader(open(mar, 'rb')).compression_type() == 'xz'
+    result = MarReader(open(mar, 'rb')).compression_type == 'xz'
+    if result:
+        log.info("%s is lzma compressed", mar)
+    else:
+        log.info("%s is not lzma compressed", mar)
+    return result
 
 
 @redo.retriable()
 def download(url, dest, mode=None):
     log.debug("Downloading %s to %s", url, dest)
     r = requests.get(url)
     r.raise_for_status()
 
@@ -69,17 +74,17 @@ def download(url, dest, mode=None):
 
 def unpack(work_env, mar, dest_dir):
     os.mkdir(dest_dir)
     unwrap_cmd = sh.Command(os.path.join(work_env.workdir,
                                          "unwrap_full_update.pl"))
     log.debug("Unwrapping %s", mar)
     env = work_env.env
     if not is_lzma_compressed_mar(mar):
-        env['MAR_OLD_FORMAT'] = 1
+        env['MAR_OLD_FORMAT'] = '1'
     elif 'MAR_OLD_FORMAT' in env:
         del env['MAR_OLD_FORMAT']
     out = unwrap_cmd(mar, _cwd=dest_dir, _env=env, _timeout=240,
                      _err_to_out=True)
     if out:
         log.debug(out)
 
 
@@ -99,23 +104,23 @@ def get_option(directory, filename, sect
     config = ConfigParser.ConfigParser()
     config.read(f)
     rv = config.get(section, option)
     log.debug("Found %s", rv)
     return rv
 
 
 def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids,
-                     version):
+                     version, use_old_format):
     log.debug("Generating partial %s", dest_mar)
     env = work_env.env
     env["MOZ_PRODUCT_VERSION"] = version
     env["MOZ_CHANNEL_ID"] = channel_ids
-    if not is_lzma_compressed_mar(dest_mar):
-        env['MAR_OLD_FORMAT'] = 1
+    if use_old_format:
+        env['MAR_OLD_FORMAT'] = '1'
     elif 'MAR_OLD_FORMAT' in env:
         del env['MAR_OLD_FORMAT']
     make_incremental_update = os.path.join(work_env.workdir,
                                            "make_incremental_update.sh")
     out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir,
                   _cwd=work_env.workdir, _env=env, _timeout=900,
                   _err_to_out=True)
     if out:
@@ -197,22 +202,22 @@ def main():
     args = parser.parse_args()
 
     logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                         level=args.log_level)
     task = json.load(args.task_definition)
     # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
 
     signing_certs = {
-        'sha1': args.sha1_signing_cert,
-        'sha384': args.sha384_signing_cert,
+        'sha1': open(args.sha1_signing_cert, 'rb').read(),
+        'sha384': open(args.sha384_signing_cert, 'rb').read(),
     }
 
-    assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048)
-    assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096)
+    assert(get_keysize(signing_certs['sha1']) == 2048)
+    assert(get_keysize(signing_certs['sha384']) == 4096)
 
     if args.no_freshclam:
         log.info("Skipping freshclam")
     else:
         log.info("Refreshing clamav db...")
         try:
             redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
                                             _timeout=300, _err_to_out=True))
@@ -223,24 +228,27 @@ def main():
     for e in task["extra"]["funsize"]["partials"]:
         for mar in (e["from_mar"], e["to_mar"]):
             verify_allowed_url(mar)
 
         work_env = WorkEnv()
         # TODO: run setup once
         work_env.setup()
         complete_mars = {}
+        use_old_format = False
         for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
             dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
             unpack_dir = os.path.join(work_env.workdir, mar_type)
             download(f, dest)
             if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
                 verify_signature(dest, signing_certs)
             complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
             complete_mars["%s_hash" % mar_type] = get_hash(dest)
+            if mar_type == 'to' and not is_lzma_compressed_mar(dest):
+                use_old_format = True
             unpack(work_env, dest, unpack_dir)
             log.info("AV-scanning %s ...", unpack_dir)
             sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
             log.info("Done.")
 
         path = os.path.join(work_env.workdir, "to")
         from_path = os.path.join(work_env.workdir, "from")
         mar_data = {
@@ -280,17 +288,18 @@ def main():
         mar_name = args.filename_template.format(**mar_data)
         mar_data["mar"] = mar_name
         dest_mar = os.path.join(work_env.workdir, mar_name)
         # TODO: download these once
         work_env.download_buildsystem_bits(repo=mar_data["repo"],
                                            revision=mar_data["revision"])
         generate_partial(work_env, from_path, path, dest_mar,
                          mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
-                         mar_data["version"])
+                         mar_data["version"],
+                         use_old_format)
         mar_data["size"] = os.path.getsize(dest_mar)
         mar_data["hash"] = get_hash(dest_mar)
 
         shutil.copy(dest_mar, args.artifacts_dir)
         work_env.cleanup()
         manifest.append(mar_data)
     manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
     with open(manifest_file, "w") as fp: