Bug 1351031 - Update rust-build container. r?mshal
Update rust-build docker image source to v0.4.3. This includes
some recent to how verification and taskcluster work to maintain
working scripts, and some changes for the newer cargo-building-
cargo build system, which aren't sufficient for bootstrapping.
MozReview-Commit-ID: 4IdbKVvco8m
--- a/taskcluster/docker/rust-build/Dockerfile
+++ b/taskcluster/docker/rust-build/Dockerfile
@@ -8,17 +8,17 @@ USER root
# Install tooltool directly from github.
ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
RUN chmod +rx /build/tooltool.py
# Add build scripts.
ADD fetch_rust.sh build_rust.sh /build/
ADD fetch_cargo.sh build_cargo.sh /build/
ADD package_rust.sh upload_rust.sh /build/
-ADD repack_rust.py /build/
+ADD repack_rust.py splat_rust.py /build/
RUN chmod +x /build/*
# Create user for doing the build.
ENV USER worker
ENV HOME /home/${USER}
RUN useradd -d ${HOME} -m ${USER}
--- a/taskcluster/docker/rust-build/VERSION
+++ b/taskcluster/docker/rust-build/VERSION
@@ -1,1 +1,1 @@
-0.4.2
+0.4.3
--- a/taskcluster/docker/rust-build/build_cargo.sh
+++ b/taskcluster/docker/rust-build/build_cargo.sh
@@ -1,20 +1,30 @@
#!/bin/bash -vex
set -x -e
: WORKSPACE ${WORKSPACE:=/home/worker}
+: BRANCH ${BRANCH:=0.15.0}
set -v
# Configure and build cargo.
if test $(uname -s) = "Darwin"; then
export MACOSX_DEPLOYMENT_TARGET=10.7
fi
+# Build the initial cargo checkout, which can download a snapshot.
pushd ${WORKSPACE}/cargo
./configure --prefix=${WORKSPACE}/rustc --local-rust-root=${WORKSPACE}/rustc
make
make dist
make install
popd
+
+# Build the version we want.
+export PATH=$PATH:${WORKSPACE}/rustc/bin
+pushd ${WORKSPACE}/cargo
+make clean
+git checkout ${BRANCH}
+OPENSSL_DIR=/rustroot/cargo64 cargo install --root=${WORKSPACE}/rustc --force
+popd
--- a/taskcluster/docker/rust-build/build_rust.sh
+++ b/taskcluster/docker/rust-build/build_rust.sh
@@ -10,18 +10,19 @@ set -v
# Configure and build rust.
OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
OPTIONS+="--enable-debuginfo"
OPTIONS+="--release-channel=stable"
i586="i586-unknown-linux-gnu"
i686="i686-unknown-linux-gnu"
x64="x86_64-unknown-linux-gnu"
-arm="arm-linux-androideabi"
+arm_android="arm-linux-androideabi"
+x86_android="i686-linux-android"
mkdir -p ${WORKSPACE}/rust-build
pushd ${WORKSPACE}/rust-build
${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
- --target=${x64},${i586} ${OPTIONS}
+ --target=${x64},${i686} ${OPTIONS}
make -j ${CORES}
make dist
make install
popd
--- a/taskcluster/docker/rust-build/fetch_cargo.sh
+++ b/taskcluster/docker/rust-build/fetch_cargo.sh
@@ -1,16 +1,16 @@
#!/bin/bash -vex
set -x -e
# Inputs, with defaults
: REPOSITORY ${REPOSITORY:=https://github.com/rust-lang/cargo}
-: BRANCH ${BRANCH:=master}
+: BRANCH ${BRANCH:=0.14.0}
: WORKSPACE ${WORKSPACE:=/home/worker}
set -v
# Check out rust sources
SRCDIR=${WORKSPACE}/cargo
git clone --recursive $REPOSITORY -b $BRANCH ${SRCDIR}
--- a/taskcluster/docker/rust-build/repack_rust.py
+++ b/taskcluster/docker/rust-build/repack_rust.py
@@ -1,23 +1,26 @@
#!/bin/env python
'''
This script downloads and repacks official rust language builds
with the necessary tool and target support for the Firefox
build environment.
'''
+import argparse
import os.path
import re
import sys
import requests
import subprocess
import toml
+def log(msg):
+ print('repack: %s' % msg)
def fetch_file(url):
'''Download a file from the given url if it's not already present.'''
filename = os.path.basename(url)
if os.path.exists(filename):
return
r = requests.get(url, stream=True)
r.raise_for_status()
@@ -32,62 +35,60 @@ def sha256sum():
return 'shasum'
else:
return 'sha256sum'
def fetch(url):
'''Download and verify a package url.'''
base = os.path.basename(url)
- print('Fetching %s...' % base)
+ log('Fetching %s...' % base)
fetch_file(url + '.asc')
fetch_file(url)
fetch_file(url + '.sha256')
- fetch_file(url + '.asc.sha256')
- print('Verifying %s...' % base)
+ log('Verifying %s...' % base)
shasum = sha256sum()
subprocess.check_call([shasum, '-c', base + '.sha256'])
- subprocess.check_call([shasum, '-c', base + '.asc.sha256'])
subprocess.check_call(['gpg', '--verify', base + '.asc', base])
- if False:
+ if True:
subprocess.check_call([
- 'keybase', 'pgp', 'verify', '-d', base + '.asc', ' -i', base,
+ 'keybase', 'pgp', 'verify', '-d', base + '.asc', '-i', base,
])
def install(filename, target):
'''Run a package's installer script against the given target directory.'''
- print(' Unpacking %s...' % filename)
+ log('Unpacking %s...' % filename)
subprocess.check_call(['tar', 'xf', filename])
basename = filename.split('.tar')[0]
- # Work around bad tarball naming in 1.15 cargo packages.
+ # Work around bad tarball naming in 1.15+ cargo packages.
basename = basename.replace('cargo-beta', 'cargo-nightly')
basename = re.sub(r'cargo-0\.[\d\.]+', 'cargo-nightly', basename)
- print(' Installing %s...' % basename)
+ log('Installing %s...' % basename)
install_cmd = [os.path.join(basename, 'install.sh')]
install_cmd += ['--prefix=' + os.path.abspath(target)]
install_cmd += ['--disable-ldconfig']
subprocess.check_call(install_cmd)
- print(' Cleaning %s...' % basename)
+ log('Cleaning %s...' % basename)
subprocess.check_call(['rm', '-rf', basename])
def package(manifest, pkg, target):
'''Pull out the package dict for a particular package and target
from the given manifest.'''
version = manifest['pkg'][pkg]['version']
info = manifest['pkg'][pkg]['target'][target]
return (version, info)
def fetch_package(manifest, pkg, host):
version, info = package(manifest, pkg, host)
- print('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash']))
+ log('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash']))
if not info['available']:
- print('%s marked unavailable for %s' % (pkg, host))
+ log('%s marked unavailable for %s' % (pkg, host))
raise AssertionError
fetch(info['url'])
return info
def fetch_std(manifest, targets):
stds = []
for target in targets:
@@ -102,101 +103,116 @@ def tar_for_host(host):
tar_ext = '.tar.xz'
else:
tar_options = 'cjf'
tar_ext = '.tar.bz2'
return tar_options, tar_ext
def repack(host, targets, channel='stable', suffix=''):
- print("Repacking rust for %s..." % host)
+ log("Repacking rust for %s..." % host)
url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
req = requests.get(url)
req.raise_for_status()
manifest = toml.loads(req.content)
if manifest['manifest-version'] != '2':
- print('ERROR: unrecognized manifest version %s.' % manifest[
- 'manifest-version'])
+ log('ERROR: unrecognized manifest version %s.' %
+ manifest['manifest-version'])
return
- print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
- print('Fetching packages...')
+ log('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+ log('Fetching packages...')
rustc = fetch_package(manifest, 'rustc', host)
cargo = fetch_package(manifest, 'cargo', host)
stds = fetch_std(manifest, targets)
- print('Installing packages...')
+ log('Installing packages...')
tar_basename = 'rustc-' + host
if suffix:
tar_basename += '-' + suffix
tar_basename += '-repack'
install_dir = 'rustc'
subprocess.check_call(['rm', '-rf', install_dir])
install(os.path.basename(rustc['url']), install_dir)
install(os.path.basename(cargo['url']), install_dir)
for std in stds:
install(os.path.basename(std['url']), install_dir)
pass
- print('Tarring %s...' % tar_basename)
+ log('Tarring %s...' % tar_basename)
tar_options, tar_ext = tar_for_host(host)
subprocess.check_call(
['tar', tar_options, tar_basename + tar_ext, install_dir])
subprocess.check_call(['rm', '-rf', install_dir])
def repack_cargo(host, channel='nightly'):
- print("Repacking cargo for %s..." % host)
+ log('Repacking cargo for %s...' % host)
# Cargo doesn't seem to have a .toml manifest.
base_url = 'https://static.rust-lang.org/cargo-dist/'
req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
req.raise_for_status()
file = ''
for line in req.iter_lines():
if line.find(host) != -1:
file = line.strip()
if not file:
- print('No manifest entry for %s!' % host)
+ log('No manifest entry for %s!' % host)
return
manifest = {
'date': req.headers['Last-Modified'],
'pkg': {
'cargo': {
'version': channel,
'target': {
host: {
'url': os.path.join(base_url, file),
'hash': None,
'available': True,
},
},
},
},
}
- print('Using manifest for cargo %s.' % channel)
- print('Fetching packages...')
+ log('Using manifest for cargo %s.' % channel)
+ log('Fetching packages...')
cargo = fetch_package(manifest, 'cargo', host)
- print('Installing packages...')
+ log('Installing packages...')
install_dir = 'cargo'
subprocess.check_call(['rm', '-rf', install_dir])
install(os.path.basename(cargo['url']), install_dir)
tar_basename = 'cargo-%s-repack' % host
- print('Tarring %s...' % tar_basename)
+ log('Tarring %s...' % tar_basename)
tar_options, tar_ext = tar_for_host(host)
subprocess.check_call(
['tar', tar_options, tar_basename + tar_ext, install_dir])
subprocess.check_call(['rm', '-rf', install_dir])
# rust platform triples
android = "armv7-linux-androideabi"
android_x86 = "i686-linux-android"
linux64 = "x86_64-unknown-linux-gnu"
linux32 = "i686-unknown-linux-gnu"
mac64 = "x86_64-apple-darwin"
mac32 = "i686-apple-darwin"
win64 = "x86_64-pc-windows-msvc"
win32 = "i686-pc-windows-msvc"
+
+def args():
+ '''Read command line arguments and return options.'''
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--channel', help='Release channel to use: '
+ 'stable, beta, or nightly')
+ args = parser.parse_args()
+ if args.channel:
+ return args.channel
+ else:
+ return 'stable'
+
if __name__ == '__main__':
- repack(mac64, [mac64, mac32])
- repack(win32, [win32])
- repack(win64, [win64])
- repack(linux64, [linux64, linux32])
- repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
- repack(linux64, [linux64, android, android_x86], suffix='android-cross')
+ channel = args()
+ repack(mac64, [mac64, mac32], channel=channel)
+ repack(win32, [win32], channel=channel)
+ repack(win64, [win64], channel=channel)
+ repack(linux64, [linux64, linux32], channel=channel)
+ repack(linux64, [linux64, mac64, mac32],
+ channel=channel, suffix='mac-cross')
+ repack(linux64, [linux64, android, android_x86],
+ channel=channel, suffix='android-cross')
--- a/taskcluster/docker/rust-build/splat_rust.py
+++ b/taskcluster/docker/rust-build/splat_rust.py
@@ -14,18 +14,17 @@ from collections import OrderedDict
def load_manifest(path):
with open(path) as f:
return json.load(f, object_pairs_hook=OrderedDict)
return None
def save_manifest(manifest, path):
with open(path, 'w') as f:
- json.dump(manifest, f,
- indent=0, separators=(',', ': '))
+ json.dump(manifest, f, indent=2, separators=(',', ': '))
f.write('\n')
def replace(manifest, stanza):
key = 'rustc'
version = stanza.get('version')
for s in manifest:
if key in s.get('filename'):
--- a/taskcluster/docker/rust-build/task.json
+++ b/taskcluster/docker/rust-build/task.json
@@ -6,22 +6,22 @@
"payload": {
"image": "quay.io/rust/gecko-rust-build",
"env": {
"RUST_BRANCH": "{rust_branch}"
},
"artifacts": {
"public/rustc.tar.xz": {
"path": "/home/worker/rustc.tar.xz",
- "expires": "{artifacts_expires}",
+ "expires": "{artifact_expires}",
"type": "file"
},
"public/manifest.tt": {
"path": "/home/worker/manifest.tt",
- "expires": "{artifacts_expires}",
+ "expires": "{artifact_expires}",
"type": "file"
}
},
"features": {
"relengAPIProxy": true
},
"maxRunTime": 6000
},
--- a/taskcluster/docker/rust-build/tcbuild.py
+++ b/taskcluster/docker/rust-build/tcbuild.py
@@ -15,18 +15,16 @@ import json
import os
import shutil
import sys
import taskcluster
import tempfile
import time
import tooltool
-requests.packages.urllib3.disable_warnings()
-
def local_file(filename):
'''
Return a path to a file next to this script.
'''
return os.path.join(os.path.dirname(__file__), filename)
@@ -58,21 +56,21 @@ def fill_template(template_file, keys):
def spawn_task(queue, args):
'''
Spawn a Taskcluster task in queue using args.
'''
task_id = taskcluster.utils.slugId()
with open(local_file('task.json'), 'rb') as template:
keys = vars(args)
now = datetime.datetime.utcnow()
+ deadline = (now + datetime.timedelta(hours=2))
+ expires = (now + datetime.timedelta(days=1))
keys['task_created'] = now.isoformat() + 'Z'
- keys['task_deadline'] = (now + datetime.timedelta(
- hours=2)).isoformat() + 'Z'
- keys['artifacts_expires'] = (now + datetime.timedelta(
- days=1)).isoformat() + 'Z'
+ keys['task_deadline'] = deadline.isoformat() + 'Z'
+ keys['artifact_expires'] = expires.isoformat() + 'Z'
payload = fill_template(template, keys)
queue.createTask(task_id, payload)
print('--- %s task %s submitted ---' % (now, task_id))
return task_id
def wait_for_task(queue, task_id, initial_wait=5):
'''