Bug 1351031 - Update rust-build container. r?mshal draft
authorRalph Giles <giles@mozilla.com>
Mon, 27 Mar 2017 14:02:44 -0700
changeset 552056 a025813e69b175adb8f2c780f256452a08c8afe5
parent 552055 3e2ac22f627dd0bdc64dc57f02ba04a40ae2185d
child 621711 3a41d46b36b8f1606fb8d29c57855b5bfb42cadb
push id51231
push userbmo:giles@thaumas.net
push dateMon, 27 Mar 2017 21:54:10 +0000
reviewersmshal
bugs1351031
milestone55.0a1
Bug 1351031 - Update rust-build container. r?mshal Update rust-build docker image source to v0.4.3. This includes some recent to how verification and taskcluster work to maintain working scripts, and some changes for the newer cargo-building- cargo build system, which aren't sufficient for bootstrapping. MozReview-Commit-ID: 4IdbKVvco8m
taskcluster/docker/rust-build/Dockerfile
taskcluster/docker/rust-build/VERSION
taskcluster/docker/rust-build/build_cargo.sh
taskcluster/docker/rust-build/build_rust.sh
taskcluster/docker/rust-build/fetch_cargo.sh
taskcluster/docker/rust-build/repack_rust.py
taskcluster/docker/rust-build/splat_rust.py
taskcluster/docker/rust-build/task.json
taskcluster/docker/rust-build/tcbuild.py
--- a/taskcluster/docker/rust-build/Dockerfile
+++ b/taskcluster/docker/rust-build/Dockerfile
@@ -8,17 +8,17 @@ USER root
 # Install tooltool directly from github.
 ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
 RUN chmod +rx /build/tooltool.py
 
 # Add build scripts.
 ADD             fetch_rust.sh build_rust.sh /build/
 ADD             fetch_cargo.sh build_cargo.sh /build/
 ADD             package_rust.sh upload_rust.sh /build/
-ADD             repack_rust.py /build/
+ADD             repack_rust.py splat_rust.py /build/
 RUN             chmod +x /build/*
 
 # Create user for doing the build.
 ENV USER worker
 ENV HOME /home/${USER}
 
 RUN useradd -d ${HOME} -m ${USER}
 
--- a/taskcluster/docker/rust-build/VERSION
+++ b/taskcluster/docker/rust-build/VERSION
@@ -1,1 +1,1 @@
-0.4.2
+0.4.3
--- a/taskcluster/docker/rust-build/build_cargo.sh
+++ b/taskcluster/docker/rust-build/build_cargo.sh
@@ -1,20 +1,30 @@
 #!/bin/bash -vex
 
 set -x -e
 
 : WORKSPACE ${WORKSPACE:=/home/worker}
+: BRANCH ${BRANCH:=0.15.0}
 
 set -v
 
 # Configure and build cargo.
 
 if test $(uname -s) = "Darwin"; then
   export MACOSX_DEPLOYMENT_TARGET=10.7
 fi
 
+# Build the initial cargo checkout, which can download a snapshot.
 pushd ${WORKSPACE}/cargo
 ./configure --prefix=${WORKSPACE}/rustc --local-rust-root=${WORKSPACE}/rustc
 make
 make dist
 make install
 popd
+
+# Build the version we want.
+export PATH=$PATH:${WORKSPACE}/rustc/bin
+pushd ${WORKSPACE}/cargo
+make clean
+git checkout ${BRANCH}
+OPENSSL_DIR=/rustroot/cargo64 cargo install --root=${WORKSPACE}/rustc --force
+popd
--- a/taskcluster/docker/rust-build/build_rust.sh
+++ b/taskcluster/docker/rust-build/build_rust.sh
@@ -10,18 +10,19 @@ set -v
 
 # Configure and build rust.
 OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
 OPTIONS+="--enable-debuginfo"
 OPTIONS+="--release-channel=stable"
 i586="i586-unknown-linux-gnu"
 i686="i686-unknown-linux-gnu"
 x64="x86_64-unknown-linux-gnu"
-arm="arm-linux-androideabi"
+arm_android="arm-linux-androideabi"
+x86_android="i686-linux-android"
 
 mkdir -p ${WORKSPACE}/rust-build
 pushd ${WORKSPACE}/rust-build
 ${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
-  --target=${x64},${i586} ${OPTIONS}
+  --target=${x64},${i686} ${OPTIONS}
 make -j ${CORES}
 make dist
 make install
 popd
--- a/taskcluster/docker/rust-build/fetch_cargo.sh
+++ b/taskcluster/docker/rust-build/fetch_cargo.sh
@@ -1,16 +1,16 @@
 #!/bin/bash -vex
 
 set -x -e
 
 # Inputs, with defaults
 
 : REPOSITORY   ${REPOSITORY:=https://github.com/rust-lang/cargo}
-: BRANCH       ${BRANCH:=master}
+: BRANCH       ${BRANCH:=0.14.0}
 
 : WORKSPACE    ${WORKSPACE:=/home/worker}
 
 set -v
 
 # Check out rust sources
 SRCDIR=${WORKSPACE}/cargo
 git clone --recursive $REPOSITORY -b $BRANCH ${SRCDIR}
--- a/taskcluster/docker/rust-build/repack_rust.py
+++ b/taskcluster/docker/rust-build/repack_rust.py
@@ -1,23 +1,26 @@
 #!/bin/env python
 '''
 This script downloads and repacks official rust language builds
 with the necessary tool and target support for the Firefox
 build environment.
 '''
 
+import argparse
 import os.path
 import re
 import sys
 
 import requests
 import subprocess
 import toml
 
+def log(msg):
+    print('repack: %s' % msg)
 
 def fetch_file(url):
     '''Download a file from the given url if it's not already present.'''
     filename = os.path.basename(url)
     if os.path.exists(filename):
         return
     r = requests.get(url, stream=True)
     r.raise_for_status()
@@ -32,62 +35,60 @@ def sha256sum():
         return 'shasum'
     else:
         return 'sha256sum'
 
 
 def fetch(url):
     '''Download and verify a package url.'''
     base = os.path.basename(url)
-    print('Fetching %s...' % base)
+    log('Fetching %s...' % base)
     fetch_file(url + '.asc')
     fetch_file(url)
     fetch_file(url + '.sha256')
-    fetch_file(url + '.asc.sha256')
-    print('Verifying %s...' % base)
+    log('Verifying %s...' % base)
     shasum = sha256sum()
     subprocess.check_call([shasum, '-c', base + '.sha256'])
-    subprocess.check_call([shasum, '-c', base + '.asc.sha256'])
     subprocess.check_call(['gpg', '--verify', base + '.asc', base])
-    if False:
+    if True:
         subprocess.check_call([
-            'keybase', 'pgp', 'verify', '-d', base + '.asc', '  -i', base,
+            'keybase', 'pgp', 'verify', '-d', base + '.asc', '-i', base,
         ])
 
 
 def install(filename, target):
     '''Run a package's installer script against the given target directory.'''
-    print(' Unpacking %s...' % filename)
+    log('Unpacking %s...' % filename)
     subprocess.check_call(['tar', 'xf', filename])
     basename = filename.split('.tar')[0]
-    # Work around bad tarball naming in 1.15 cargo packages.
+    # Work around bad tarball naming in 1.15+ cargo packages.
     basename = basename.replace('cargo-beta', 'cargo-nightly')
     basename = re.sub(r'cargo-0\.[\d\.]+', 'cargo-nightly', basename)
-    print(' Installing %s...' % basename)
+    log('Installing %s...' % basename)
     install_cmd = [os.path.join(basename, 'install.sh')]
     install_cmd += ['--prefix=' + os.path.abspath(target)]
     install_cmd += ['--disable-ldconfig']
     subprocess.check_call(install_cmd)
-    print(' Cleaning %s...' % basename)
+    log('Cleaning %s...' % basename)
     subprocess.check_call(['rm', '-rf', basename])
 
 
 def package(manifest, pkg, target):
     '''Pull out the package dict for a particular package and target
     from the given manifest.'''
     version = manifest['pkg'][pkg]['version']
     info = manifest['pkg'][pkg]['target'][target]
     return (version, info)
 
 
 def fetch_package(manifest, pkg, host):
     version, info = package(manifest, pkg, host)
-    print('%s %s\n  %s\n  %s' % (pkg, version, info['url'], info['hash']))
+    log('%s %s\n  %s\n  %s' % (pkg, version, info['url'], info['hash']))
     if not info['available']:
-        print('%s marked unavailable for %s' % (pkg, host))
+        log('%s marked unavailable for %s' % (pkg, host))
         raise AssertionError
     fetch(info['url'])
     return info
 
 
 def fetch_std(manifest, targets):
     stds = []
     for target in targets:
@@ -102,101 +103,116 @@ def tar_for_host(host):
         tar_ext = '.tar.xz'
     else:
         tar_options = 'cjf'
         tar_ext = '.tar.bz2'
     return tar_options, tar_ext
 
 
 def repack(host, targets, channel='stable', suffix=''):
-    print("Repacking rust for %s..." % host)
+    log("Repacking rust for %s..." % host)
     url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
     req = requests.get(url)
     req.raise_for_status()
     manifest = toml.loads(req.content)
     if manifest['manifest-version'] != '2':
-        print('ERROR: unrecognized manifest version %s.' % manifest[
-            'manifest-version'])
+        log('ERROR: unrecognized manifest version %s.' %
+              manifest['manifest-version'])
         return
-    print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
-    print('Fetching packages...')
+    log('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+    log('Fetching packages...')
     rustc = fetch_package(manifest, 'rustc', host)
     cargo = fetch_package(manifest, 'cargo', host)
     stds = fetch_std(manifest, targets)
-    print('Installing packages...')
+    log('Installing packages...')
     tar_basename = 'rustc-' + host
     if suffix:
         tar_basename += '-' + suffix
     tar_basename += '-repack'
     install_dir = 'rustc'
     subprocess.check_call(['rm', '-rf', install_dir])
     install(os.path.basename(rustc['url']), install_dir)
     install(os.path.basename(cargo['url']), install_dir)
     for std in stds:
         install(os.path.basename(std['url']), install_dir)
         pass
-    print('Tarring %s...' % tar_basename)
+    log('Tarring %s...' % tar_basename)
     tar_options, tar_ext = tar_for_host(host)
     subprocess.check_call(
         ['tar', tar_options, tar_basename + tar_ext, install_dir])
     subprocess.check_call(['rm', '-rf', install_dir])
 
 
 def repack_cargo(host, channel='nightly'):
-    print("Repacking cargo for %s..." % host)
+    log('Repacking cargo for %s...' % host)
     # Cargo doesn't seem to have a .toml manifest.
     base_url = 'https://static.rust-lang.org/cargo-dist/'
     req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
     req.raise_for_status()
     file = ''
     for line in req.iter_lines():
         if line.find(host) != -1:
             file = line.strip()
     if not file:
-        print('No manifest entry for %s!' % host)
+        log('No manifest entry for %s!' % host)
         return
     manifest = {
         'date': req.headers['Last-Modified'],
         'pkg': {
             'cargo': {
                 'version': channel,
                 'target': {
                     host: {
                         'url': os.path.join(base_url, file),
                         'hash': None,
                         'available': True,
                     },
                 },
             },
         },
     }
-    print('Using manifest for cargo %s.' % channel)
-    print('Fetching packages...')
+    log('Using manifest for cargo %s.' % channel)
+    log('Fetching packages...')
     cargo = fetch_package(manifest, 'cargo', host)
-    print('Installing packages...')
+    log('Installing packages...')
     install_dir = 'cargo'
     subprocess.check_call(['rm', '-rf', install_dir])
     install(os.path.basename(cargo['url']), install_dir)
     tar_basename = 'cargo-%s-repack' % host
-    print('Tarring %s...' % tar_basename)
+    log('Tarring %s...' % tar_basename)
     tar_options, tar_ext = tar_for_host(host)
     subprocess.check_call(
         ['tar', tar_options, tar_basename + tar_ext, install_dir])
     subprocess.check_call(['rm', '-rf', install_dir])
 
 
 # rust platform triples
 android = "armv7-linux-androideabi"
 android_x86 = "i686-linux-android"
 linux64 = "x86_64-unknown-linux-gnu"
 linux32 = "i686-unknown-linux-gnu"
 mac64 = "x86_64-apple-darwin"
 mac32 = "i686-apple-darwin"
 win64 = "x86_64-pc-windows-msvc"
 win32 = "i686-pc-windows-msvc"
 
+
+def args():
+    '''Read command line arguments and return options.'''
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--channel', help='Release channel to use: '
+                                          'stable, beta, or nightly')
+    args = parser.parse_args()
+    if args.channel:
+        return args.channel
+    else:
+        return 'stable'
+
 if __name__ == '__main__':
-    repack(mac64, [mac64, mac32])
-    repack(win32, [win32])
-    repack(win64, [win64])
-    repack(linux64, [linux64, linux32])
-    repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
-    repack(linux64, [linux64, android, android_x86], suffix='android-cross')
+    channel = args()
+    repack(mac64, [mac64, mac32], channel=channel)
+    repack(win32, [win32], channel=channel)
+    repack(win64, [win64], channel=channel)
+    repack(linux64, [linux64, linux32], channel=channel)
+    repack(linux64, [linux64, mac64, mac32],
+           channel=channel, suffix='mac-cross')
+    repack(linux64, [linux64, android, android_x86],
+           channel=channel, suffix='android-cross')
--- a/taskcluster/docker/rust-build/splat_rust.py
+++ b/taskcluster/docker/rust-build/splat_rust.py
@@ -14,18 +14,17 @@ from collections import OrderedDict
 def load_manifest(path):
     with open(path) as f:
         return json.load(f, object_pairs_hook=OrderedDict)
     return None
 
 
 def save_manifest(manifest, path):
     with open(path, 'w') as f:
-        json.dump(manifest, f,
-                  indent=0, separators=(',', ': '))
+        json.dump(manifest, f, indent=2, separators=(',', ': '))
         f.write('\n')
 
 
 def replace(manifest, stanza):
     key = 'rustc'
     version = stanza.get('version')
     for s in manifest:
         if key in s.get('filename'):
--- a/taskcluster/docker/rust-build/task.json
+++ b/taskcluster/docker/rust-build/task.json
@@ -6,22 +6,22 @@
     "payload": {
         "image": "quay.io/rust/gecko-rust-build",
         "env": {
             "RUST_BRANCH": "{rust_branch}"
         },
         "artifacts": {
             "public/rustc.tar.xz": {
               "path": "/home/worker/rustc.tar.xz",
-              "expires": "{artifacts_expires}",
+              "expires": "{artifact_expires}",
               "type": "file"
             },
             "public/manifest.tt": {
               "path": "/home/worker/manifest.tt",
-              "expires": "{artifacts_expires}",
+              "expires": "{artifact_expires}",
               "type": "file"
             }
         },
         "features": {
           "relengAPIProxy": true
         },
         "maxRunTime": 6000
     },
--- a/taskcluster/docker/rust-build/tcbuild.py
+++ b/taskcluster/docker/rust-build/tcbuild.py
@@ -15,18 +15,16 @@ import json
 import os
 import shutil
 import sys
 import taskcluster
 import tempfile
 import time
 import tooltool
 
-requests.packages.urllib3.disable_warnings()
-
 
 def local_file(filename):
     '''
     Return a path to a file next to this script.
     '''
     return os.path.join(os.path.dirname(__file__), filename)
 
 
@@ -58,21 +56,21 @@ def fill_template(template_file, keys):
 def spawn_task(queue, args):
     '''
     Spawn a Taskcluster task in queue using args.
     '''
     task_id = taskcluster.utils.slugId()
     with open(local_file('task.json'), 'rb') as template:
         keys = vars(args)
         now = datetime.datetime.utcnow()
+        deadline = (now + datetime.timedelta(hours=2))
+        expires = (now + datetime.timedelta(days=1))
         keys['task_created'] = now.isoformat() + 'Z'
-        keys['task_deadline'] = (now + datetime.timedelta(
-            hours=2)).isoformat() + 'Z'
-        keys['artifacts_expires'] = (now + datetime.timedelta(
-            days=1)).isoformat() + 'Z'
+        keys['task_deadline'] = deadline.isoformat() + 'Z'
+        keys['artifact_expires'] = expires.isoformat() + 'Z'
         payload = fill_template(template, keys)
     queue.createTask(task_id, payload)
     print('--- %s task %s submitted ---' % (now, task_id))
     return task_id
 
 
 def wait_for_task(queue, task_id, initial_wait=5):
     '''