Bug 1321863 - Update gecko-rust-build to v0.4.0. r?dustin draft
authorRalph Giles <giles@mozilla.com>
Fri, 02 Dec 2016 11:16:53 -0800
changeset 447138 622c33d9e154e1d3f2da3c9636942f2acc68d742
parent 447079 e756ee86e3ea1225d8cdf38ca0eb8b0ded15dd41
child 538981 8a18bb868912e4406589bc26d0cc86d5d46b676a
push id38000
push userbmo:giles@thaumas.net
push dateFri, 02 Dec 2016 19:20:49 +0000
reviewersdustin
bugs1321863
milestone53.0a1
Bug 1321863 - Update gecko-rust-build to v0.4.0. r?dustin Sync with upstream. Have been using the repack script for that last few updates. - New splat_rust.py script to update gecko tooltool manifests. - Drop cargo nightly repacks in favor of stable cargo. - Build with linux32 to i586. - Switch linux32 back to i686 in repacks. - Add android-x86. MozReview-Commit-ID: 6pGq7oAZV1v
testing/docker/rust-build/Dockerfile
testing/docker/rust-build/VERSION
testing/docker/rust-build/build_rust.sh
testing/docker/rust-build/repack_rust.py
testing/docker/rust-build/splat_rust.py
testing/docker/rust-build/tcbuild.py
--- a/testing/docker/rust-build/Dockerfile
+++ b/testing/docker/rust-build/Dockerfile
@@ -1,21 +1,16 @@
 FROM          quay.io/rust/rust-buildbot
 MAINTAINER    Ralph Giles <giles@mozilla.com>
 
 # Reset user/workdir from parent image so we can install software.
 WORKDIR /
 USER root
 
-# Update base.
-RUN yum upgrade -y
-RUN yum clean all
-
 # Install tooltool directly from github.
-RUN mkdir /builds
 ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
 RUN chmod +rx /build/tooltool.py
 
 # Add build scripts.
 ADD             fetch_rust.sh build_rust.sh /build/
 ADD             fetch_cargo.sh build_cargo.sh /build/
 ADD             package_rust.sh upload_rust.sh /build/
 ADD             repack_rust.py /build/
--- a/testing/docker/rust-build/VERSION
+++ b/testing/docker/rust-build/VERSION
@@ -1,1 +1,1 @@
-0.3.0
+0.4.0
--- a/testing/docker/rust-build/build_rust.sh
+++ b/testing/docker/rust-build/build_rust.sh
@@ -7,20 +7,21 @@ set -x -e
 CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
 
 set -v
 
 # Configure and build rust.
 OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
 OPTIONS+="--enable-debuginfo"
 OPTIONS+="--release-channel=stable"
-x32="i686-unknown-linux-gnu"
+i586="i586-unknown-linux-gnu"
+i686="i686-unknown-linux-gnu"
 x64="x86_64-unknown-linux-gnu"
 arm="arm-linux-androideabi"
 
 mkdir -p ${WORKSPACE}/rust-build
 pushd ${WORKSPACE}/rust-build
 ${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
-  --target=${x64},${x32} ${OPTIONS}
+  --target=${x64},${i586} ${OPTIONS}
 make -j ${CORES}
 make dist
 make install
 popd
--- a/testing/docker/rust-build/repack_rust.py
+++ b/testing/docker/rust-build/repack_rust.py
@@ -28,20 +28,21 @@ def fetch(url):
   fetch_file(url + '.asc')
   fetch_file(url)
   fetch_file(url + '.sha256')
   fetch_file(url + '.asc.sha256')
   print('Verifying %s...' % base)
   subprocess.check_call(['shasum', '-c', base + '.sha256'])
   subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
   subprocess.check_call(['gpg', '--verify', base + '.asc', base])
-  subprocess.check_call(['keybase', 'pgp', 'verify',
-      '-d', base + '.asc',
-      '-i', base,
-  ])
+  if False:
+      subprocess.check_call(['keybase', 'pgp', 'verify',
+          '-d', base + '.asc',
+          '-i', base,
+      ])
 
 def install(filename, target):
   '''Run a package's installer script against the given target directory.'''
   print(' Unpacking %s...' % filename)
   subprocess.check_call(['tar', 'xf', filename])
   basename = filename.split('.tar')[0]
   print(' Installing %s...' % basename)
   install_cmd = [os.path.join(basename, 'install.sh')]
@@ -152,26 +153,23 @@ def repack_cargo(host, channel='nightly'
   tar_basename = 'cargo-%s-repack' % host
   print('Tarring %s...' % tar_basename)
   tar_options, tar_ext = tar_for_host(host)
   subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
   subprocess.check_call(['rm', '-rf', install_dir])
 
 # rust platform triples
 android="armv7-linux-androideabi"
+android_x86="i686-linux-android"
 linux64="x86_64-unknown-linux-gnu"
 linux32="i686-unknown-linux-gnu"
 mac64="x86_64-apple-darwin"
 mac32="i686-apple-darwin"
 win64="x86_64-pc-windows-msvc"
 win32="i686-pc-windows-msvc"
 
 if __name__ == '__main__':
   repack(mac64, [mac64, mac32])
   repack(win32, [win32])
   repack(win64, [win64])
   repack(linux64, [linux64, linux32])
   repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
-  repack(linux64, [linux64, android], suffix='android-cross')
-  repack_cargo(mac64)
-  repack_cargo(win32)
-  repack_cargo(win64)
-  repack_cargo(linux64)
+  repack(linux64, [linux64, android, android_x86], suffix='android-cross')
new file mode 100644
--- /dev/null
+++ b/testing/docker/rust-build/splat_rust.py
@@ -0,0 +1,102 @@
+#!/bin/env python
+'''
+This script patches tooltool manifests in the firefox source
+tree to update them to a new set of rust packages.
+'''
+
+import json
+import os.path
+import sys
+
+from collections import OrderedDict
+
+def load_manifest(path):
+    with open(path) as f:
+        return json.load(f, object_pairs_hook=OrderedDict)
+    return None
+
+def save_manifest(manifest, path):
+    with open(path, 'w') as f:
+        json.dump(manifest, f,
+                indent=0, separators=(',', ': '))
+        f.write('\n')
+
+def replace(manifest, stanza):
+    key = 'rustc'
+    version = stanza.get('version')
+    for s in manifest:
+        if key in s.get('filename'):
+            if version:
+                print('Replacing %s\n     with %s' % (s['version'], version))
+                s['version'] = version
+            print('  old %s' % s['digest'][:12])
+            s['digest'] = stanza['digest']
+            s['size'] = stanza['size']
+            print('  new %s' % s['digest'][:12])
+            return True
+    print('Warning: Could not find matching %s filename' %key)
+    return False
+
+def update_manifest(source_manifest, target, target_filename):
+    for stanza in source_manifest:
+        filename = stanza.get('filename')
+        if target in filename:
+            size = int(stanza.get('size'))
+            print('Found %s %d bytes' % (filename, size))
+            version = stanza.get('version')
+            if version:
+                print('  %s' % version)
+            print('Updating %s' % target_filename)
+            old = load_manifest(target_filename)
+            replace(old, stanza)
+            save_manifest(old, target_filename)
+            break
+
+'''Mapping from targets to target filenames.'''
+TARGETS = {
+        'x86_64-unknown-linux-gnu-repack': [
+            'b2g/dev/config/tooltool-manifests/linux64/hazard.manifest',
+            'b2g/dev/config/tooltool-manifests/linux64/releng.manifest',
+            'browser/config/tooltool-manifests/linux32/releng.manifest',
+            'browser/config/tooltool-manifests/linux64/asan.manifest',
+            'browser/config/tooltool-manifests/linux64/clang.manifest',
+            'browser/config/tooltool-manifests/linux64/clang.manifest.centos6',
+            'browser/config/tooltool-manifests/linux64/hazard.manifest',
+            'browser/config/tooltool-manifests/linux64/msan.manifest',
+            'browser/config/tooltool-manifests/linux64/releng.manifest',
+            ],
+        'x86_64-unknown-linux-gnu-android-cross-repack': [
+            'mobile/android/config/tooltool-manifests/android/releng.manifest',
+            'mobile/android/config/tooltool-manifests/android-x86/releng.manifest',
+            'mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest',
+            ],
+        'x86_64-unknown-linux-gnu-mac-cross-repack': [
+            'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
+            ],
+        'x86_64-apple-darwin-repack': [
+            'browser/config/tooltool-manifests/macosx64/clang.manifest',
+            'browser/config/tooltool-manifests/macosx64/releng.manifest',
+            ],
+        'x86_64-pc-windows-msvc-repack': [
+            'browser/config/tooltool-manifests/win64/clang.manifest',
+            'browser/config/tooltool-manifests/win64/releng.manifest',
+            ],
+        'i686-pc-windows-msvc-repack': [
+            'browser/config/tooltool-manifests/win32/build-clang-cl.manifest',
+            'browser/config/tooltool-manifests/win32/clang.manifest',
+            'browser/config/tooltool-manifests/win32/releng.manifest',
+            ],
+}
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('%s PATH' % sys.argv[0])
+        sys.exit(1)
+
+    base_path = sys.argv[1]
+
+    updates = load_manifest('manifest.tt')
+    for target, filenames in TARGETS.items():
+        for target_filename in filenames:
+            update_manifest(updates, target,
+                    os.path.join(base_path, target_filename))
--- a/testing/docker/rust-build/tcbuild.py
+++ b/testing/docker/rust-build/tcbuild.py
@@ -3,17 +3,16 @@
 This script triggers a taskcluster task, waits for it to finish,
 fetches the artifacts, uploads them to tooltool, and updates
 the in-tree tooltool manifests.
 '''
 
 from __future__ import print_function
 
 import requests.packages.urllib3
-requests.packages.urllib3.disable_warnings()
 
 import argparse
 import datetime
 import json
 import os
 import shutil
 import sys
 import taskcluster
@@ -165,17 +164,17 @@ def upload_to_tooltool(tooltool_auth, ta
             '--authentication-file', tooltool_auth,
             '--message', 'Built from taskcluster task {}'.format(task_id),
         ])
         return manifest
     finally:
         os.chdir(oldcwd)
 
 def update_manifest(artifact, manifest, local_gecko_clone):
-    platform = linux
+    platform = 'linux'
     manifest_dir = os.path.join(local_gecko_clone,
                                 'testing', 'config', 'tooltool-manifests')
     platform_dir = [p for p in os.listdir(manifest_dir)
                     if p.startswith(platform)][0]
     tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest')
     print('%s -> %s' % (manifest, tree_manifest))
     shutil.copyfile(manifest, tree_manifest)