Bug 1430908 - Use --progress=dot:mega with wget; r?glandium draft
authorGregory Szorc <gps@mozilla.com>
Tue, 16 Jan 2018 14:24:31 -0800
changeset 721190 720b27953bd0cfc45d31c0a8ac751e452317ba97
parent 720975 9be7249e74fd7f6d9163b59d3386ed01038197a0
child 746267 4c5ee684a14d3bd2cebe0855687cfbeb6fd3fe60
push id95766
push userbmo:gps@mozilla.com
push dateTue, 16 Jan 2018 22:24:45 +0000
reviewersglandium
bugs1430908
milestone59.0a1
Bug 1430908 - Use --progress=dot:mega with wget; r?glandium By default, wget prints dots every 1k bytes. This can render a lot of output for large files. We switch to the "mega" style, which makes each dot represent 64k, thus reducing output by up to 64x. We also force the use of dot display. By default, it uses "bar" which attempts to use terminal formatting if possible. Since most of this code executes in CI and terminal control characters can interfere with logged output, we force the use of "dot." (Although wget appears to automatically switch to dot in TC today. But consistency is good.) MozReview-Commit-ID: IpTWJdcauTV
build/unix/build-binutils/build-binutils.sh
build/unix/build-gcc/download-tools.sh
build/unix/build-gtk3/build-gtk3.sh
build/unix/build-hfsplus/build-hfsplus.sh
taskcluster/docker/recipes/install-node.sh
taskcluster/scripts/misc/build-mingw32-nsis.sh
taskcluster/scripts/misc/build-upx.sh
taskcluster/scripts/misc/repack-proguard-jar.sh
--- a/build/unix/build-binutils/build-binutils.sh
+++ b/build/unix/build-binutils/build-binutils.sh
@@ -9,17 +9,17 @@ if [ -z "$root_dir" -o ! -d "$root_dir" 
 fi
 cd $root_dir
 
 if test -z $TMPDIR; then
   TMPDIR=/tmp/
 fi
 
 # Download the source of the specified version of binutils
-wget -c -P $TMPDIR ftp://ftp.gnu.org/gnu/binutils/binutils-${binutils_version}.tar.bz2 || exit 1
+wget -c --progress=dot:mega -P $TMPDIR ftp://ftp.gnu.org/gnu/binutils/binutils-${binutils_version}.tar.bz2 || exit 1
 tar xjf $TMPDIR/binutils-${binutils_version}.tar.bz2
 
 # Build binutils
 mkdir binutils-objdir
 cd binutils-objdir
 
 ../binutils-$binutils_version/configure --prefix /tools/binutils/ --enable-gold --enable-plugins --disable-nls || exit 1
 make $make_flags || exit 1
--- a/build/unix/build-gcc/download-tools.sh
+++ b/build/unix/build-gcc/download-tools.sh
@@ -12,17 +12,17 @@ if test -z $TMPDIR; then
 fi
 
 mkdir $root_dir/gpg
 GPG="gpg --homedir $root_dir/gpg"
 
 > $root_dir/downloads
 
 download() {
-  wget -c -P $TMPDIR $1/$2
+  wget -c --progress=dot:mega -P $TMPDIR $1/$2
   (cd $TMPDIR; sha256sum $2) >> $root_dir/downloads
 }
 
 download_and_check() {
   download $1 ${2%.*}
-  wget -c -P $TMPDIR $1/$2
+  wget -c --progress=dot:mega -P $TMPDIR $1/$2
   $GPG --verify $TMPDIR/$2 $TMPDIR/${2%.*}
-}
\ No newline at end of file
+}
--- a/build/unix/build-gtk3/build-gtk3.sh
+++ b/build/unix/build-gtk3/build-gtk3.sh
@@ -33,17 +33,17 @@ make_flags=-j$(nproc)
 yum install -y libtool-ltdl-devel libtool-ltdl-devel.i686 json-c-devel json-c-devel.i686 libsndfile-devel libsndfile-devel.i686
 
 build() {
 	name=$1
 	shift
 	pkg=$(echo $name | tr '+-' '__')
 	version=$(eval echo \$${pkg}_version)
 	url=$(eval echo \$${pkg}_url)
-	wget -c -P $root_dir $url
+	wget -c --progress=dot:mega -P $root_dir $url
 	tar -axf $root_dir/$name-$version.tar.*
 	mkdir -p build/$name
 	cd build/$name
 	eval ../../$name-$version/configure --disable-static $* $configure_args --libdir=/usr/local/$lib
 	make $make_flags
 	make install
 	find /usr/local/$lib -name \*.la -delete
 	cd ../..
--- a/build/unix/build-hfsplus/build-hfsplus.sh
+++ b/build/unix/build-hfsplus/build-hfsplus.sh
@@ -19,17 +19,17 @@ if test -z $TMPDIR; then
 fi
 
 # Set an md5 check file to validate input
 echo "${md5sum} *${TMPDIR}/${filename}" > $TMPDIR/hfsplus.MD5
 
 # Most-upstream is https://opensource.apple.com/source/diskdev_cmds/
 
 # Download the source of the specified version of hfsplus
-wget -c -P $TMPDIR http://pkgs.fedoraproject.org/repo/pkgs/hfsplus-tools/${filename}/${md5sum}/${filename} || exit 1
+wget -c --progress=dot:mega -P $TMPDIR http://pkgs.fedoraproject.org/repo/pkgs/hfsplus-tools/${filename}/${md5sum}/${filename} || exit 1
 md5sum -c $TMPDIR/hfsplus.MD5 || exit 1
 mkdir hfsplus-source
 tar xzf $TMPDIR/${filename} -C hfsplus-source --strip-components=1
 
 # Build
 cd hfsplus-source
 # We want to statically link against libcrypto. On CentOS, that requires zlib
 # and libdl, because of FIPS functions pulling in more than necessary from
--- a/taskcluster/docker/recipes/install-node.sh
+++ b/taskcluster/docker/recipes/install-node.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This script installs Node v6.
 
-wget https://nodejs.org/dist/v6.9.1/node-v6.9.1-linux-x64.tar.gz
+wget --progress=dot:mega https://nodejs.org/dist/v6.9.1/node-v6.9.1-linux-x64.tar.gz
 echo 'a9d9e6308931fa2a2b0cada070516d45b76d752430c31c9198933c78f8d54b17  node-v6.9.1-linux-x64.tar.gz' | sha256sum -c
 tar -C /usr/local -xz --strip-components 1 < node-v6.9.1-linux-x64.tar.gz
 node -v  # verify
 npm -v
--- a/taskcluster/scripts/misc/build-mingw32-nsis.sh
+++ b/taskcluster/scripts/misc/build-mingw32-nsis.sh
@@ -82,17 +82,17 @@ EOF
 # --------------
 
 download_and_check http://zlib.net/ zlib-1.2.11.tar.gz.asc
 tar xaf $TMPDIR/zlib-1.2.11.tar.gz
 cd zlib-1.2.11
 make -f win32/Makefile.gcc PREFIX=i686-w64-mingw32-
 
 cd ..
-wget https://downloads.sourceforge.net/project/nsis/NSIS%203/3.01/nsis-3.01-src.tar.bz2
+wget --progress=dot:mega https://downloads.sourceforge.net/project/nsis/NSIS%203/3.01/nsis-3.01-src.tar.bz2
 echo "559703cc25f78697be1784a38d1d9a19c97d27a200dc9257d1483c028c6be9242cbcd10391ba618f88561c2ba57fdbd8b3607bea47ed8c3ad7509a6ae4075138  nsis-3.01-src.tar.bz2" | sha512sum -c -
 bunzip2 nsis-3.01-src.tar.bz2
 tar xaf nsis-3.01-src.tar
 cd nsis-3.01-src
 # I don't know how to make the version work with the environment variables/config flags the way the author appears to
 sed -i "s/'VERSION', 'Version of NSIS', cvs_version/'VERSION', 'Version of NSIS', '3.01'/" SConstruct
 scons XGCC_W32_PREFIX=i686-w64-mingw32- ZLIB_W32=../zlib-1.2.11 SKIPUTILS="NSIS Menu" PREFIX=$INSTALL_DIR/ install
 
--- a/taskcluster/scripts/misc/build-upx.sh
+++ b/taskcluster/scripts/misc/build-upx.sh
@@ -7,17 +7,17 @@ INSTALL_DIR=$WORKSPACE/upx
 UPLOAD_DIR=$HOME/artifacts
 
 mkdir -p $INSTALL_DIR/bin
 
 cd $WORKSPACE
 
 # --------------
 
-wget http://www.oberhumer.com/opensource/ucl/download/ucl-1.03.tar.gz
+wget --progress=dot:mega http://www.oberhumer.com/opensource/ucl/download/ucl-1.03.tar.gz
 echo "5847003d136fbbca1334dd5de10554c76c755f7c  ucl-1.03.tar.gz" | sha1sum -c -
 tar xf ucl-1.03.tar.gz
 cd ucl-1.03
 ./configure
 make -j$(nproc)
 
 # --------------
 
--- a/taskcluster/scripts/misc/repack-proguard-jar.sh
+++ b/taskcluster/scripts/misc/repack-proguard-jar.sh
@@ -12,17 +12,17 @@ VERSION=5.3.3
 URL=https://newcontinuum.dl.sourceforge.net/project/proguard/proguard/5.3/proguard$VERSION.tar.gz
 ARCHIVE=proguard$VERSION.tar.gz
 DIR=proguard$VERSION
 SHA256SUM=95bf9580107f00d0e26f01026dcfe9e7a772e5449488b03ba832836c3760b3af
 
 mkdir -p $UPLOAD_DIR $STAGE
 
 cd $WORKSPACE
-wget $URL
+wget --progress=dot:mega $URL
 echo "$SHA256SUM  $ARCHIVE" | sha256sum -c -
 
 # Just the file we need.
 tar --wildcards -zxvf $ARCHIVE '*/proguard.jar'
 
 # The archive is to satisfy source distribution requirements.
 mv $ARCHIVE $UPLOAD_DIR