Bug 1423539 Update funsize docker image and python3 r=rail draft
authorSimon Fraser <sfraser@mozilla.com>
Wed, 06 Dec 2017 12:00:50 +0000
changeset 708258 b56b659dac3783a519b77b89110a8c87336c5eb2
parent 707931 f2b3bed28c503dd0b77330646d8cb3e1f0dbbf93
child 708311 6d0b488c55044e757d8aa47c67b63bba7e46cda9
child 708312 9eadda186a50aa1db5828edbdf37975318325397
child 708314 dac6d8f77fc6795d2d4f2ca5ba12e859c2efa9fc
child 708315 3ba6a46d5abfbdeb1e29798c1d8e46b81ca68734
push id92348
push usersfraser@mozilla.com
push dateWed, 06 Dec 2017 14:20:37 +0000
reviewersrail
bugs1423539
milestone59.0a1
Bug 1423539 Update funsize docker image and python3 r=rail MozReview-Commit-ID: 2iWrYevwtvs
taskcluster/docker/funsize-update-generator/Dockerfile
taskcluster/docker/funsize-update-generator/scripts/funsize.py
taskcluster/docker/funsize-update-generator/scripts/recompress.py
--- a/taskcluster/docker/funsize-update-generator/Dockerfile
+++ b/taskcluster/docker/funsize-update-generator/Dockerfile
@@ -1,30 +1,30 @@
-FROM ubuntu:vivid
-MAINTAINER Rail Aliiev <rail@mozilla.com>
+FROM ubuntu:xenial
+MAINTAINER Simon Fraser <sfraser@mozilla.com>
 
 # Required software
 ENV DEBIAN_FRONTEND noninteractive
 # Chain apt-get commands with apt-get clean in a single docker RUN
 # to make sure that files are removed within a single docker layer
 RUN apt-get update -q && \
     apt-get install -yyq --no-install-recommends \
-    python python-setuptools python-cryptography  libgetopt-simple-perl \
-    bzip2 clamav clamav-freshclam python-requests python-sh curl \
-    python-dev gcc liblzma-dev xz-utils && \
+    python3.5 python3-setuptools python3-cryptography libgetopt-simple-perl \
+    bzip2 clamav clamav-freshclam python3-requests python3-sh curl \
+    python3-dev gcc liblzma-dev xz-utils && \
     apt-get clean
 RUN useradd -d /home/worker -s /bin/bash -m worker
 COPY requirements.txt /tmp/
 
 # Freshclam may be flaky, retry if it fails
 RUN for i in 1 2 3 4 5; do freshclam --verbose && break || sleep 15; done
 
 # python-pip installs a lot of dependencies increasing the size of an image
 # drastically. Using easy_install saves us almost 200M.
-RUN easy_install pip
+RUN easy_install3 pip
 RUN pip install -r /tmp/requirements.txt
 
 # scripts
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
 
 COPY runme.sh /runme.sh
 COPY recompress.sh /recompress.sh
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -1,16 +1,14 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
-
-import ConfigParser
+import configparser
 import argparse
 import functools
 import hashlib
 import json
 import logging
 import os
 import shutil
 import tempfile
@@ -103,17 +101,17 @@ def find_file(directory, filename):
             log.debug("Found %s", f)
             return f
 
 
 def get_option(directory, filename, section, option):
     log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory,
               filename)
     f = find_file(directory, filename)
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     config.read(f)
     rv = config.get(section, option)
     log.debug("Found %s", rv)
     return rv
 
 
 def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids,
                      version, use_old_format):
@@ -132,18 +130,17 @@ def generate_partial(work_env, from_dir,
                   _err_to_out=True)
     if out:
         log.debug(out)
 
 
 def get_hash(path, hash_type="sha512"):
     h = hashlib.new(hash_type)
     with open(path, "rb") as f:
-        for chunk in iter(functools.partial(f.read, 4096), ''):
-            h.update(chunk)
+        h.update(f.read())
     return h.hexdigest()
 
 
 class WorkEnv(object):
 
     def __init__(self):
         self.workdir = tempfile.mkdtemp()
 
@@ -203,18 +200,18 @@ def main():
                         default=DEFAULT_FILENAME_TEMPLATE)
     parser.add_argument("--no-freshclam", action="store_true", default=False,
                         help="Do not refresh ClamAV DB")
     parser.add_argument("-q", "--quiet", dest="log_level",
                         action="store_const", const=logging.WARNING,
                         default=logging.DEBUG)
     args = parser.parse_args()
 
-    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
-                        level=args.log_level)
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
+    log.setLevel(args.log_level)
     task = json.load(args.task_definition)
     # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
 
     signing_certs = {
         'sha1': open(args.sha1_signing_cert, 'rb').read(),
         'sha384': open(args.sha384_signing_cert, 'rb').read(),
     }
 
--- a/taskcluster/docker/funsize-update-generator/scripts/recompress.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/recompress.py
@@ -1,12 +1,11 @@
-#!/usr/bin/env python
-from __future__ import absolute_import, print_function
+#!/usr/bin/env python3
 
-import ConfigParser
+import configparser
 import argparse
 import functools
 import hashlib
 import json
 import logging
 import os
 import shutil
 import tempfile
@@ -38,17 +37,17 @@ def find_file(directory, filename):
             log.debug("Found %s", f)
             return f
 
 
 def get_option(directory, filename, section, option):
     log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory,
               filename)
     f = find_file(directory, filename)
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     config.read(f)
     rv = config.get(section, option)
     log.debug("Found %s", rv)
     return rv
 
 
 def verify_signature(mar, certs):
     log.info("Checking %s signature", mar)