Bug 1321863 - Update gecko-rust-build to v0.4.0. r?dustin
Sync with upstream. Have been using the repack script for
that last few updates.
- New splat_rust.py script to update gecko tooltool manifests.
- Drop cargo nightly repacks in favor of stable cargo.
- Build with linux32 to i586.
- Switch linux32 back to i686 in repacks.
- Add android-x86.
MozReview-Commit-ID: 6pGq7oAZV1v
--- a/testing/docker/rust-build/Dockerfile
+++ b/testing/docker/rust-build/Dockerfile
@@ -1,21 +1,16 @@
FROM quay.io/rust/rust-buildbot
MAINTAINER Ralph Giles <giles@mozilla.com>
# Reset user/workdir from parent image so we can install software.
WORKDIR /
USER root
-# Update base.
-RUN yum upgrade -y
-RUN yum clean all
-
# Install tooltool directly from github.
-RUN mkdir /builds
ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
RUN chmod +rx /build/tooltool.py
# Add build scripts.
ADD fetch_rust.sh build_rust.sh /build/
ADD fetch_cargo.sh build_cargo.sh /build/
ADD package_rust.sh upload_rust.sh /build/
ADD repack_rust.py /build/
--- a/testing/docker/rust-build/VERSION
+++ b/testing/docker/rust-build/VERSION
@@ -1,1 +1,1 @@
-0.3.0
+0.4.0
--- a/testing/docker/rust-build/build_rust.sh
+++ b/testing/docker/rust-build/build_rust.sh
@@ -7,20 +7,21 @@ set -x -e
CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
set -v
# Configure and build rust.
OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
OPTIONS+="--enable-debuginfo"
OPTIONS+="--release-channel=stable"
-x32="i686-unknown-linux-gnu"
+i586="i586-unknown-linux-gnu"
+i686="i686-unknown-linux-gnu"
x64="x86_64-unknown-linux-gnu"
arm="arm-linux-androideabi"
mkdir -p ${WORKSPACE}/rust-build
pushd ${WORKSPACE}/rust-build
${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
- --target=${x64},${x32} ${OPTIONS}
+ --target=${x64},${i586} ${OPTIONS}
make -j ${CORES}
make dist
make install
popd
--- a/testing/docker/rust-build/repack_rust.py
+++ b/testing/docker/rust-build/repack_rust.py
@@ -28,20 +28,21 @@ def fetch(url):
fetch_file(url + '.asc')
fetch_file(url)
fetch_file(url + '.sha256')
fetch_file(url + '.asc.sha256')
print('Verifying %s...' % base)
subprocess.check_call(['shasum', '-c', base + '.sha256'])
subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
subprocess.check_call(['gpg', '--verify', base + '.asc', base])
- subprocess.check_call(['keybase', 'pgp', 'verify',
- '-d', base + '.asc',
- '-i', base,
- ])
+ if False:
+ subprocess.check_call(['keybase', 'pgp', 'verify',
+ '-d', base + '.asc',
+ '-i', base,
+ ])
def install(filename, target):
'''Run a package's installer script against the given target directory.'''
print(' Unpacking %s...' % filename)
subprocess.check_call(['tar', 'xf', filename])
basename = filename.split('.tar')[0]
print(' Installing %s...' % basename)
install_cmd = [os.path.join(basename, 'install.sh')]
@@ -152,26 +153,23 @@ def repack_cargo(host, channel='nightly'
tar_basename = 'cargo-%s-repack' % host
print('Tarring %s...' % tar_basename)
tar_options, tar_ext = tar_for_host(host)
subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
subprocess.check_call(['rm', '-rf', install_dir])
# rust platform triples
android="armv7-linux-androideabi"
+android_x86="i686-linux-android"
linux64="x86_64-unknown-linux-gnu"
linux32="i686-unknown-linux-gnu"
mac64="x86_64-apple-darwin"
mac32="i686-apple-darwin"
win64="x86_64-pc-windows-msvc"
win32="i686-pc-windows-msvc"
if __name__ == '__main__':
repack(mac64, [mac64, mac32])
repack(win32, [win32])
repack(win64, [win64])
repack(linux64, [linux64, linux32])
repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
- repack(linux64, [linux64, android], suffix='android-cross')
- repack_cargo(mac64)
- repack_cargo(win32)
- repack_cargo(win64)
- repack_cargo(linux64)
+ repack(linux64, [linux64, android, android_x86], suffix='android-cross')
new file mode 100644
--- /dev/null
+++ b/testing/docker/rust-build/splat_rust.py
@@ -0,0 +1,102 @@
+#!/bin/env python
+'''
+This script patches tooltool manifests in the firefox source
+tree to update them to a new set of rust packages.
+'''
+
+import json
+import os.path
+import sys
+
+from collections import OrderedDict
+
+def load_manifest(path):
+ with open(path) as f:
+ return json.load(f, object_pairs_hook=OrderedDict)
+ return None
+
+def save_manifest(manifest, path):
+ with open(path, 'w') as f:
+ json.dump(manifest, f,
+ indent=0, separators=(',', ': '))
+ f.write('\n')
+
+def replace(manifest, stanza):
+ key = 'rustc'
+ version = stanza.get('version')
+ for s in manifest:
+ if key in s.get('filename'):
+ if version:
+ print('Replacing %s\n with %s' % (s['version'], version))
+ s['version'] = version
+ print(' old %s' % s['digest'][:12])
+ s['digest'] = stanza['digest']
+ s['size'] = stanza['size']
+ print(' new %s' % s['digest'][:12])
+ return True
+ print('Warning: Could not find matching %s filename' %key)
+ return False
+
+def update_manifest(source_manifest, target, target_filename):
+ for stanza in source_manifest:
+ filename = stanza.get('filename')
+ if target in filename:
+ size = int(stanza.get('size'))
+ print('Found %s %d bytes' % (filename, size))
+ version = stanza.get('version')
+ if version:
+ print(' %s' % version)
+ print('Updating %s' % target_filename)
+ old = load_manifest(target_filename)
+ replace(old, stanza)
+ save_manifest(old, target_filename)
+ break
+
+'''Mapping from targets to target filenames.'''
+TARGETS = {
+ 'x86_64-unknown-linux-gnu-repack': [
+ 'b2g/dev/config/tooltool-manifests/linux64/hazard.manifest',
+ 'b2g/dev/config/tooltool-manifests/linux64/releng.manifest',
+ 'browser/config/tooltool-manifests/linux32/releng.manifest',
+ 'browser/config/tooltool-manifests/linux64/asan.manifest',
+ 'browser/config/tooltool-manifests/linux64/clang.manifest',
+ 'browser/config/tooltool-manifests/linux64/clang.manifest.centos6',
+ 'browser/config/tooltool-manifests/linux64/hazard.manifest',
+ 'browser/config/tooltool-manifests/linux64/msan.manifest',
+ 'browser/config/tooltool-manifests/linux64/releng.manifest',
+ ],
+ 'x86_64-unknown-linux-gnu-android-cross-repack': [
+ 'mobile/android/config/tooltool-manifests/android/releng.manifest',
+ 'mobile/android/config/tooltool-manifests/android-x86/releng.manifest',
+ 'mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest',
+ ],
+ 'x86_64-unknown-linux-gnu-mac-cross-repack': [
+ 'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
+ ],
+ 'x86_64-apple-darwin-repack': [
+ 'browser/config/tooltool-manifests/macosx64/clang.manifest',
+ 'browser/config/tooltool-manifests/macosx64/releng.manifest',
+ ],
+ 'x86_64-pc-windows-msvc-repack': [
+ 'browser/config/tooltool-manifests/win64/clang.manifest',
+ 'browser/config/tooltool-manifests/win64/releng.manifest',
+ ],
+ 'i686-pc-windows-msvc-repack': [
+ 'browser/config/tooltool-manifests/win32/build-clang-cl.manifest',
+ 'browser/config/tooltool-manifests/win32/clang.manifest',
+ 'browser/config/tooltool-manifests/win32/releng.manifest',
+ ],
+}
+
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print('%s PATH' % sys.argv[0])
+ sys.exit(1)
+
+ base_path = sys.argv[1]
+
+ updates = load_manifest('manifest.tt')
+ for target, filenames in TARGETS.items():
+ for target_filename in filenames:
+ update_manifest(updates, target,
+ os.path.join(base_path, target_filename))
--- a/testing/docker/rust-build/tcbuild.py
+++ b/testing/docker/rust-build/tcbuild.py
@@ -3,17 +3,16 @@
This script triggers a taskcluster task, waits for it to finish,
fetches the artifacts, uploads them to tooltool, and updates
the in-tree tooltool manifests.
'''
from __future__ import print_function
import requests.packages.urllib3
-requests.packages.urllib3.disable_warnings()
import argparse
import datetime
import json
import os
import shutil
import sys
import taskcluster
@@ -165,17 +164,17 @@ def upload_to_tooltool(tooltool_auth, ta
'--authentication-file', tooltool_auth,
'--message', 'Built from taskcluster task {}'.format(task_id),
])
return manifest
finally:
os.chdir(oldcwd)
def update_manifest(artifact, manifest, local_gecko_clone):
- platform = linux
+ platform = 'linux'
manifest_dir = os.path.join(local_gecko_clone,
'testing', 'config', 'tooltool-manifests')
platform_dir = [p for p in os.listdir(manifest_dir)
if p.startswith(platform)][0]
tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest')
print('%s -> %s' % (manifest, tree_manifest))
shutil.copyfile(manifest, tree_manifest)