Bug 1439860 update-packaging linting fixes and shell style updates r=sledru,bhearsum
MozReview-Commit-ID: 2vMbmonkVwD
--- a/tools/lint/shellcheck.yml
+++ b/tools/lint/shellcheck.yml
@@ -1,12 +1,13 @@
---
shellcheck:
description: Shell script linter
include:
- taskcluster/docker/
+ - tools/update-packaging/
exclude: []
# 1090: https://github.com/koalaman/shellcheck/wiki/SC1090
# 'Can't follow a non-constant source'
extensions: ['sh']
excludecodes: ['1090', '1091']
type: external
payload: shell:lint
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -8,50 +8,48 @@
# Author: Darin Fisher
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
MBSDIFF=${MBSDIFF:-mbsdiff}
if [[ -z "${MAR_OLD_FORMAT}" ]]; then
- XZ=${XZ:-xz}
- $XZ --version > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- # If $XZ is not set and not found on the path then this is probably
- # running on a windows buildbot. Some of the Windows build systems have
- # xz.exe in topsrcdir/xz/. Look in the places this would be in both a
- # mozilla-central and comm-central build.
- XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
- $XZ --version > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
- $XZ --version > /dev/null 2>&1
- if [ $? -ne 0 ]; then
- echo "xz was not found on this system!"
- echo "exiting"
- exit 1
- fi
+ while read -r XZ
+ do
+ if ${XZ} --version > /dev/null 2>&1
+ then
+ break
fi
+ done << EOM
+${XZ:-xz}
+$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe
+$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe
+EOM
+
+ if [ -z "${XZ}" ]
+ then
+ echo "xz was not found on this system!"
+ echo "exiting"
+ exit 1
fi
else
MAR_OLD_FORMAT=1
BZIP2=${BZIP2:-bzip2}
fi
# -----------------------------------------------------------------------------
# Helper routines
notice() {
echo "$*" 1>&2
}
get_file_size() {
- info=($(ls -ln "$1"))
- echo ${info[4]}
+ stat -f"%z" "$1"
}
copy_perm() {
reference="$1"
target="$2"
if [ -x "$reference" ]; then
chmod 0755 "$target"
@@ -66,163 +64,167 @@ make_add_instruction() {
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console
- if [ $4 ]; then
+ if [ "$4" ]; then
forced=" (forced)"
else
forced=
fi
- is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
- if [ $is_extension = "1" ]; then
+ if [[ "$f" =~ distribution/extensions/.*/ ]]
+ then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
- echo "add-if \"$testdir\" \"$f\"" >> "$filev2"
- if [ ! $filev3 = "" ]; then
- echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
+ printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}"
+ if [ -n "${filev3}" ]; then
+ printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}"
fi
else
notice " add \"$f\"$forced"
- echo "add \"$f\"" >> "$filev2"
+ printf 'add "%s"\n' "${f}" >> "${filev2}"
if [ ! "$filev3" = "" ]; then
- echo "add \"$f\"" >> "$filev3"
+ printf 'add "%s"\n' "${f}" >> "${filev3}"
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
- if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
- `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
- ## "true" *giggle*
+ if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \
+ [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
+ then
return 0;
fi
- ## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
- if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
- ## "true" *giggle*
+ if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
+ then
return 0;
fi
- ## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
- echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
+ printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}"
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
- is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
- if [ $is_extension = "1" ]; then
+ if [[ "$f" =~ distribution/extensions/.*/ ]]
+ then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2"
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
+ printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}"
+ printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}"
else
notice " patch \"$f.patch\" \"$f\""
- echo "patch \"$f.patch\" \"$f\"" >> "$filev2"
- echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
+ printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}"
+ printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}"
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
- if [ -f "$dir/removed-files" ]; then
+ if [ -f "$dir/removed-files" ]
+ then
listfile="$dir/removed-files"
- elif [ -f "$dir/Contents/Resources/removed-files" ]; then
+ elif [ -f "$dir/Contents/Resources/removed-files" ]
+ then
listfile="$dir/Contents/Resources/removed-files"
fi
- if [ -n "$listfile" ]; then
- # Map spaces to pipes so that we correctly handle filenames with spaces.
- files=($(cat "$listfile" | tr " " "|" | sort -r))
- num_files=${#files[*]}
- for ((i=0; $i<$num_files; i=$i+1)); do
- # Map pipes back to whitespace and remove carriage returns
- f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
- # Trim whitespace
- f=$(echo $f)
- # Exclude blank lines.
- if [ -n "$f" ]; then
- # Exclude comments
- if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
- if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
- notice " rmdir \"$f\""
- echo "rmdir \"$f\"" >> "$filev2"
- echo "rmdir \"$f\"" >> "$filev3"
- elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
- # Remove the *
- f=$(echo "$f" | sed -e 's:\*$::')
- notice " rmrfdir \"$f\""
- echo "rmrfdir \"$f\"" >> "$filev2"
- echo "rmrfdir \"$f\"" >> "$filev3"
- else
- notice " remove \"$f\""
- echo "remove \"$f\"" >> "$filev2"
- echo "remove \"$f\"" >> "$filev3"
- fi
- fi
- fi
- done
+ if [ -z "$listfile" ]; then
+ return
fi
+
+ while read -r f
+ do
+ # skip blank lines
+ if [ -z "$f" ]
+ then
+ continue
+ fi
+ # skip comments
+ if [[ "$f" =~ ^\#.* ]]
+ then
+ continue
+ fi
+
+ if [[ "$f" =~ .*/$ ]]
+ then
+ notice " rmdir \"$f\""
+ printf 'rmdir "%s"\n' "${f}" >> "${filev2}"
+ printf 'rmdir "%s"\n' "${f}" >> "${filev3}"
+ elif [[ "$f" =~ .*/\*$ ]]
+ then
+ # Remove the *
+ f=${f%\*}
+ notice " rmrfdir \"$f\""
+ printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}"
+ printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}"
+ else
+ notice " remove \"$f\""
+ printf 'remove "%s"\n' "${f}" >> "${filev2}"
+ printf 'remove "%s"\n' "${f}" >> "${filev3}"
+ fi
+ done <"$listfile"
}
# List all files in the current directory, stripping leading "./"
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
-
+ tmpfile="$(mktemp)"
find . -type f \
! -name "update.manifest" \
! -name "updatev2.manifest" \
! -name "updatev3.manifest" \
! -name "temp-dirlist" \
! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \
- | sort -r > "temp-filelist"
- while read file; do
+ | sort -r > "${tmpfile}"
+ while read -r file; do
eval "${1}[$count]=\"$file\""
(( count++ ))
- done < "temp-filelist"
- rm "temp-filelist"
+ done <"${tmpfile}"
+ rm -f "${tmpfile}"
}
# List all directories in the current directory, stripping leading "./"
list_dirs() {
count=0
+ tmpfile="$(mktemp)"
find . -type d \
! -name "." \
! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \
- | sort -r > "temp-dirlist"
- while read dir; do
+ | sort -r > "${tmpfile}"
+ while read -r dir; do
eval "${1}[$count]=\"$dir\""
(( count++ ))
- done < "temp-dirlist"
- rm "temp-dirlist"
+ done <"${tmpfile}"
+ rm -f "${tmpfile}"
}
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -3,90 +3,90 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool generates full update packages for the update system.
# Author: Darin Fisher
#
-. $(dirname "$0")/common.sh
+# shellcheck disable=SC1090
+. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
- notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
+ notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY"
}
if [ $# = 0 ]; then
print_usage
exit 1
fi
-if [ $1 = -h ]; then
+if [ "$1" = -h ]; then
print_usage
notice ""
notice "The contents of DIRECTORY will be stored in ARCHIVE."
notice ""
notice "Options:"
notice " -h show this help text"
notice ""
exit 1
fi
# -----------------------------------------------------------------------------
archive="$1"
targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
-if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then
# Remove the /
- targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+ targetdir=${targetdir%/}
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
-pushd "$targetdir"
+pushd "$targetdir" || exit 1
if test $? -ne 0 ; then
exit 1
fi
if [ ! -f "precomplete" ]; then
if [ ! -f "Contents/Resources/precomplete" ]; then
notice "precomplete file is missing!"
exit 1
fi
fi
+declare -a files
list_files files
-popd
+popd || exit 1
# Add the type of update to the beginning of the update manifests.
-> "$updatemanifestv2"
-> "$updatemanifestv3"
+: > "$updatemanifestv2"
+: > "$updatemanifestv3"
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
echo "type \"complete\"" >> "$updatemanifestv2"
echo "type \"complete\"" >> "$updatemanifestv3"
notice ""
notice "Adding file add instructions to update manifests"
-num_files=${#files[*]}
-for ((i=0; $i<$num_files; i=$i+1)); do
- f="${files[$i]}"
-
+for f in "${files[@]}"
+do
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -3,22 +3,23 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool generates incremental update packages for the update system.
# Author: Darin Fisher
#
-. $(dirname "$0")/common.sh
+# shellcheck disable=SC1090
+. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
- notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
+ notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE FROMDIR TODIR"
notice ""
notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
notice ""
notice "Options:"
notice " -h show this help text"
notice " -f clobber this file in the installation"
notice " Must be a path to a file to clobber in the partial update."
notice ""
@@ -26,58 +27,49 @@ print_usage() {
check_for_forced_update() {
force_list="$1"
forced_file_chk="$2"
local f
if [ "$forced_file_chk" = "precomplete" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "removed-files" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "chrome.manifest" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "$forced_file_chk" = "Contents/Resources/chrome.manifest" ]; then
- ## "true" *giggle*
return 0;
fi
if [ "${forced_file_chk##*.}" = "chk" ]; then
- ## "true" *giggle*
return 0;
fi
for f in $force_list; do
#echo comparing $forced_file_chk to $f
if [ "$forced_file_chk" = "$f" ]; then
- ## "true" *giggle*
return 0;
fi
done
- ## 'false'... because this is bash. Oh yay!
return 1;
}
if [ $# = 0 ]; then
print_usage
exit 1
fi
@@ -92,102 +84,95 @@ do
;;
?) print_usage; exit 1
;;
esac
done
# -----------------------------------------------------------------------------
-let arg_start=$OPTIND-1
+arg_start=$((OPTIND-1))
shift $arg_start
archive="$1"
olddir="$2"
newdir="$3"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
-if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
- # Remove the /
- newdir=$(echo "$newdir" | sed -e 's:\/$::')
-fi
+
+# Remove the /
+newdir="${newdir%/}"
+
workdir="$newdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
archivefiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
-pushd "$olddir"
-if test $? -ne 0 ; then
+pushd "$olddir" || exit 1
+
+declare -a oldfiles
+list_files oldfiles
+declare -a olddirs
+list_dirs olddirs
+
+popd || exit 1
+
+pushd "$newdir" || exit 1
+
+if [ ! -f "precomplete" ] && [ ! -f "Contents/Resources/precomplete" ]
+then
+ notice "precomplete file is missing!"
exit 1
fi
-list_files oldfiles
-list_dirs olddirs
-
-popd
-
-pushd "$newdir"
-if test $? -ne 0 ; then
- exit 1
-fi
-
-if [ ! -f "precomplete" ]; then
- if [ ! -f "Contents/Resources/precomplete" ]; then
- notice "precomplete file is missing!"
- exit 1
- fi
-fi
-
-list_dirs newdirs
+declare -a newfiles
list_files newfiles
-popd
+popd || exit 1
# Add the type of update to the beginning of the update manifests.
notice ""
notice "Adding type instruction to update manifests"
-> $updatemanifestv2
-> $updatemanifestv3
+: > "$updatemanifestv2"
+: > "$updatemanifestv3"
notice " type partial"
-echo "type \"partial\"" >> $updatemanifestv2
-echo "type \"partial\"" >> $updatemanifestv3
+echo 'type "partial"' >> "$updatemanifestv2"
+echo 'type "partial"' >> "$updatemanifestv3"
notice ""
notice "Adding file patch and add instructions to update manifests"
-num_oldfiles=${#oldfiles[*]}
-remove_array=
+declare -a remove_array
num_removes=0
-for ((i=0; $i<$num_oldfiles; i=$i+1)); do
- f="${oldfiles[$i]}"
-
+for f in "${oldfiles[@]}"
+do
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
- mkdir -p `dirname "$workdir/$f"`
+ mkdir -p "$(dirname "$workdir/$f")"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
continue 1
fi
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
- mkdir -p `dirname "$workdir/$f"`
+ mkdir -p "$(dirname "$workdir/$f")"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
@@ -207,61 +192,51 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
# An example of MBSDIFF_HOOK env variable could look like this:
# export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
# where myscript.sh has the following usage:
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
#
# Note: patches are bzipped or xz stashed in funsize to gain more speed
- # if service is not enabled then default to old behavior
- if [ -z "$MBSDIFF_HOOK" ]; then
- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- if [[ -n $MAR_OLD_FORMAT ]]; then
- $BZIP2 -z9 "$workdir/$f.patch"
- else
- $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
- fi
+ if [[ -n $MAR_OLD_FORMAT ]]
+ then
+ COMPRESS="$BZIP2 -z9"
+ EXT=".bz2"
+ STDOUT="-c"
else
- # if service enabled then check patch existence for retrieval
- if [[ -n $MAR_OLD_FORMAT ]]; then
- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
- notice "file \"$f\" found in funsize, diffing skipped"
- else
- # if not found already - compute it and cache it for future use
- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $BZIP2 -z9 "$workdir/$f.patch"
- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
- fi
+ COMPRESS="$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force"
+ EXT=".xz"
+ STDOUT="--stdout"
+ fi
+
+ if [ -z "$MBSDIFF_HOOK" ]
+ then
+ $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+ $COMPRESS "$workdir/$f.patch"
+ else
+ if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}"
+ then
+ notice "file \"$f\" found in funsize, diffing skipped"
else
- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
- notice "file \"$f\" found in funsize, diffing skipped"
- else
- # if not found already - compute it and cache it for future use
- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
- fi
+ # if not found already - compute it and cache it for future use
+ $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+ $COMPRESS "$workdir/$f.patch"
+ $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch${EXT}"
fi
fi
- if [[ -n $MAR_OLD_FORMAT ]]; then
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
- else
- $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
- fi
+ $COMPRESS $STDOUT "$newdir/$f" > "$workdir/$f"
+
copy_perm "$newdir/$f" "$workdir/$f"
- if [[ -n $MAR_OLD_FORMAT ]]; then
- patchfile="$workdir/$f.patch.bz2"
- else
- patchfile="$workdir/$f.patch.xz"
- fi
+ patchfile="$workdir/$f.patch${EXT}"
+
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
- if [ $patchsize -lt $fullsize ]; then
+ if [ "$patchsize" -lt "$fullsize" ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
archivefiles="$archivefiles \"$f.patch\""
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
rm -f "$patchfile"
archivefiles="$archivefiles \"$f\""
@@ -273,28 +248,26 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
remove_array[$num_removes]=$f
(( num_removes++ ))
fi
done
# Newly added files
notice ""
notice "Adding file add instructions to update manifests"
-num_newfiles=${#newfiles[*]}
-for ((i=0; $i<$num_newfiles; i=$i+1)); do
- f="${newfiles[$i]}"
-
- # If we've already tested this file, then skip it
- for ((j=0; $j<$num_oldfiles; j=$j+1)); do
- if [ "$f" = "${oldfiles[j]}" ]; then
+for f in "${newfiles[@]}"
+do
+ for j in "${oldfiles[@]}"
+ do
+ if [ "$f" = "$j" ]
+ then
continue 2
fi
done
-
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
@@ -307,39 +280,38 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
fi
archivefiles="$archivefiles \"$f\""
done
notice ""
notice "Adding file remove instructions to update manifests"
-for ((i=0; $i<$num_removes; i=$i+1)); do
- f="${remove_array[$i]}"
+for f in "${remove_array[@]}"
+do
notice " remove \"$f\""
- echo "remove \"$f\"" >> $updatemanifestv2
- echo "remove \"$f\"" >> $updatemanifestv3
+ echo "remove \"$f\"" >> "$updatemanifestv2"
+ echo "remove \"$f\"" >> "$updatemanifestv3"
done
# Add remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
notice ""
notice "Adding directory remove instructions for directories that no longer exist"
-num_olddirs=${#olddirs[*]}
-for ((i=0; $i<$num_olddirs; i=$i+1)); do
- f="${olddirs[$i]}"
+for f in "${olddirs[@]}"
+do
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/"
- echo "rmdir \"$f/\"" >> $updatemanifestv2
- echo "rmdir \"$f/\"" >> $updatemanifestv3
+ echo "rmdir \"$f/\"" >> "$updatemanifestv2"
+ echo "rmdir \"$f/\"" >> "$updatemanifestv3"
fi
done
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
else
$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
--- a/tools/update-packaging/test/buildrefmars.sh
+++ b/tools/update-packaging/test/buildrefmars.sh
@@ -3,25 +3,25 @@
if [ -f "ref.mar" ]; then
rm "ref.mar"
fi
if [ -f "ref-mac.mar" ]; then
rm "ref-mac.mar"
fi
- ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
- ../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac
+ ../make_incremental_update.sh ref.mar "$(pwd)/from" "$(pwd)/to"
+ ../make_incremental_update.sh ref-mac.mar "$(pwd)/from-mac" "$(pwd)/to-mac"
if [ -f "product-1.0.lang.platform.complete.mar" ]; then
rm "product-1.0.lang.platform.complete.mar"
fi
if [ -f "product-2.0.lang.platform.complete.mar" ]; then
rm "product-2.0.lang.platform.complete.mar"
fi
if [ -f "product-2.0.lang.mac.complete.mar" ]; then
rm "product-2.0.lang.mac.complete.mar"
fi
-./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
-./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
-./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac"
-./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
+./make_full_update.sh product-1.0.lang.platform.complete.mar "$(pwd)/from"
+./make_full_update.sh product-2.0.lang.platform.complete.mar "$(pwd)/to"
+./make_full_update.sh product-1.0.lang.mac.complete.mar "$(pwd)/from-mac"
+./make_full_update.sh product-2.0.lang.mac.complete.mar "$(pwd)/to-mac"
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -1,14 +1,14 @@
#!/bin/bash
# helper tool for testing. Cats the manifest out of a mar file
mar="$1"
workdir="/tmp/catmanifest"
rm -rf "$workdir"
mkdir -p "$workdir"
-cp "$1" "$workdir"
-cd "$workdir"
-mar -x "$1"
+cp "$mar" "$workdir"
+cd "$workdir" || exit 1
+mar -x "$mar"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
cat updatev2.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -2,34 +2,54 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
-# In here to use the local common.sh to allow the full mars to have unfiltered files
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
-XZ=${XZ:-xz}
MBSDIFF=${MBSDIFF:-mbsdiff}
+if [[ -z "${MAR_OLD_FORMAT}" ]]; then
+ while read -r XZ
+ do
+ if ${XZ} --version > /dev/null 2>&1
+ then
+ break
+ fi
+ done << EOM
+${XZ:-xz}
+$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe
+$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe
+EOM
+
+ if [ -z "${XZ}" ]
+ then
+ echo "xz was not found on this system!"
+ echo "exiting"
+ exit 1
+ fi
+else
+ MAR_OLD_FORMAT=1
+ BZIP2=${BZIP2:-bzip2}
+fi
# -----------------------------------------------------------------------------
# Helper routines
notice() {
echo "$*" 1>&2
}
get_file_size() {
- info=($(ls -ln "$1"))
- echo ${info[4]}
+ stat -f"%z" "$1"
}
copy_perm() {
reference="$1"
target="$2"
if [ -x "$reference" ]; then
chmod 0755 "$target"
@@ -44,159 +64,167 @@ make_add_instruction() {
# The third param will be an empty string when a file add instruction is only
# needed in the version 2 manifest. This only happens when the file has an
# add-if-not instruction in the version 3 manifest. This is due to the
# precomplete file prior to the version 3 manifest having a remove instruction
# for this file so the file is removed before applying a complete update.
filev3="$3"
# Used to log to the console
- if [ $4 ]; then
+ if [ "$4" ]; then
forced=" (forced)"
else
forced=
fi
- is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
- if [ $is_extension = "1" ]; then
+ if [[ "$f" =~ distribution/extensions/.*/ ]]
+ then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
- echo "add-if \"$testdir\" \"$f\"" >> $filev2
- if [ ! $filev3 = "" ]; then
- echo "add-if \"$testdir\" \"$f\"" >> $filev3
+ printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev2}"
+ if [ -n "${filev3}" ]; then
+ printf 'add-if "%s" "%s"\n' "${testdir}" "${f}" >> "${filev3}"
fi
else
notice " add \"$f\"$forced"
- echo "add \"$f\"" >> $filev2
- if [ ! $filev3 = "" ]; then
- echo "add \"$f\"" >> $filev3
+ printf 'add "%s"\n' "${f}" >> "${filev2}"
+ if [ ! "$filev3" = "" ]; then
+ printf 'add "%s"\n' "${f}" >> "${filev3}"
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
- if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
- `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
- ## "true" *giggle*
+ if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" ] || \
+ [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
+ then
return 0;
fi
- ## 'false'... because this is bash. Oh yay!
return 1;
}
check_for_add_to_manifestv2() {
add_if_not_file_chk="$1"
- if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then
- ## "true" *giggle*
+ if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]
+ then
return 0;
fi
- ## 'false'... because this is bash. Oh yay!
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
- echo "add-if-not \"$f\" \"$f\"" >> $filev3
+ printf 'add-if-not "%s" "%s"\n' "${f}" "${f}" >> "${filev3}"
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
- is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
- if [ $is_extension = "1" ]; then
+ if [[ "$f" =~ distribution/extensions/.*/ ]]
+ then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
+ printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev2}"
+ printf 'patch-if "%s" "%s.patch" "%s"\n' "${testdir}" "${f}" "${f}" >> "${filev3}"
else
notice " patch \"$f.patch\" \"$f\""
- echo "patch \"$f.patch\" \"$f\"" >> $filev2
- echo "patch \"$f.patch\" \"$f\"" >> $filev3
+ printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev2}"
+ printf 'patch "%s.patch" "%s"\n' "${f}" "${f}" >> "${filev3}"
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
- if [ -f "$dir/removed-files" ]; then
+ if [ -f "$dir/removed-files" ]
+ then
listfile="$dir/removed-files"
- elif [ -f "$dir/Contents/Resources/removed-files" ]; then
+ elif [ -f "$dir/Contents/Resources/removed-files" ]
+ then
listfile="$dir/Contents/Resources/removed-files"
fi
- if [ -n "$listfile" ]; then
- # Map spaces to pipes so that we correctly handle filenames with spaces.
- files=($(cat "$listfile" | tr " " "|" | sort -r))
- num_files=${#files[*]}
- for ((i=0; $i<$num_files; i=$i+1)); do
- # Map pipes back to whitespace and remove carriage returns
- f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
- # Trim whitespace
- f=$(echo $f)
- # Exclude blank lines.
- if [ -n "$f" ]; then
- # Exclude comments
- if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
- if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
- notice " rmdir \"$f\""
- echo "rmdir \"$f\"" >> $filev2
- echo "rmdir \"$f\"" >> $filev3
- elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
- # Remove the *
- f=$(echo "$f" | sed -e 's:\*$::')
- notice " rmrfdir \"$f\""
- echo "rmrfdir \"$f\"" >> $filev2
- echo "rmrfdir \"$f\"" >> $filev3
- else
- notice " remove \"$f\""
- echo "remove \"$f\"" >> $filev2
- echo "remove \"$f\"" >> $filev3
- fi
- fi
- fi
- done
+ if [ -z "$listfile" ]; then
+ return
fi
+
+ while read -r f
+ do
+ # skip blank lines
+ if [ -z "$f" ]
+ then
+ continue
+ fi
+ # skip comments
+ if [[ "$f" =~ ^\#.* ]]
+ then
+ continue
+ fi
+
+ if [[ "$f" =~ .*/$ ]]
+ then
+ notice " rmdir \"$f\""
+ printf 'rmdir "%s"\n' "${f}" >> "${filev2}"
+ printf 'rmdir "%s"\n' "${f}" >> "${filev3}"
+ elif [[ "$f" =~ .*/\*$ ]]
+ then
+ # Remove the *
+ f=${f%\*}
+ notice " rmrfdir \"$f\""
+ printf 'rmrfdir "%s"\n' "${f}" >> "${filev2}"
+ printf 'rmrfdir "%s"\n' "${f}" >> "${filev3}"
+ else
+ notice " remove \"$f\""
+ printf 'remove "%s"\n' "${f}" >> "${filev2}"
+ printf 'remove "%s"\n' "${f}" >> "${filev3}"
+ fi
+ done <"$listfile"
}
# List all files in the current directory, stripping leading "./"
# Pass a variable name and it will be filled as an array.
list_files() {
count=0
-
- # Removed the exclusion cases here to allow for generation of testing mars
+ tmpfile="$(mktemp)"
find . -type f \
+ ! -name "update.manifest" \
+ ! -name "updatev2.manifest" \
+ ! -name "updatev3.manifest" \
+ ! -name "temp-dirlist" \
+ ! -name "temp-filelist" \
| sed 's/\.\/\(.*\)/\1/' \
- | sort -r > "$workdir/temp-filelist"
- while read file; do
+ | sort -r > "${tmpfile}"
+ while read -r file; do
eval "${1}[$count]=\"$file\""
(( count++ ))
- done < "$workdir/temp-filelist"
- rm "$workdir/temp-filelist"
+ done <"${tmpfile}"
+ rm -f "${tmpfile}"
}
# List all directories in the current directory, stripping leading "./"
list_dirs() {
count=0
+ tmpfile="$(mktemp)"
find . -type d \
! -name "." \
! -name ".." \
| sed 's/\.\/\(.*\)/\1/' \
- | sort -r > "$workdir/temp-dirlist"
- while read dir; do
+ | sort -r > "${tmpfile}"
+ while read -r dir; do
eval "${1}[$count]=\"$dir\""
(( count++ ))
- done < "$workdir/temp-dirlist"
- rm "$workdir/temp-dirlist"
+ done <"${tmpfile}"
+ rm -f "${tmpfile}"
}
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -7,45 +7,45 @@ testDir="$3"
workdir="/tmp/diffmar/$testDir"
fromdir="$workdir/0"
todir="$workdir/1"
# On Windows, creation time can be off by a second or more between the files in
# the fromdir and todir due to them being extracted synchronously so use
# time-style and exclude seconds from the creation time.
lsargs="-algR"
-unamestr=`uname`
+unamestr=$(uname)
if [ ! "$unamestr" = 'Darwin' ]; then
- unamestr=`uname -o`
- if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then
+ unamestr=$(uname -o)
+ if [ "$unamestr" = 'Msys' ] || [ "$unamestr" = "Cygwin" ]; then
lsargs="-algR --time-style=+%Y-%m-%d-%H:%M"
fi
fi
rm -rf "$workdir"
mkdir -p "$fromdir"
mkdir -p "$todir"
cp "$1" "$fromdir"
cp "$2" "$todir"
-cd "$fromdir"
-mar -x "$1"
-rm "$1"
+cd "$fromdir" || exit 1
+mar -x "$marA"
+rm "$marA"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
-ls $lsargs > files.txt
+ls "$lsargs" > files.txt
-cd "$todir"
-mar -x "$2"
-rm "$2"
+cd "$todir" || exit 1
+mar -x "$marB"
+rm "$marB"
mv updatev2.manifest updatev2.manifest.xz
xz -d updatev2.manifest.xz
mv updatev3.manifest updatev3.manifest.xz
xz -d updatev3.manifest.xz
-ls $lsargs > files.txt
+ls "$lsargs" > files.txt
echo "diffing $fromdir and $todir"
echo "on linux shell sort and python sort return different results"
echo "which can cause differences in the manifest files"
diff -ru "$fromdir" "$todir"
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -4,90 +4,90 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool generates full update packages for the update system.
# Author: Darin Fisher
#
# In here to use the local common.sh to allow the full mars to have unfiltered files
-. $(dirname "$0")/common.sh
+# shellcheck disable=SC1090
+. "$(dirname "$0")/common.sh"
# -----------------------------------------------------------------------------
print_usage() {
- notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
+ notice "Usage: $(basename "$0") [OPTIONS] ARCHIVE DIRECTORY"
}
if [ $# = 0 ]; then
print_usage
exit 1
fi
-if [ $1 = -h ]; then
+if [ "$1" = -h ]; then
print_usage
notice ""
notice "The contents of DIRECTORY will be stored in ARCHIVE."
notice ""
notice "Options:"
notice " -h show this help text"
notice ""
exit 1
fi
# -----------------------------------------------------------------------------
archive="$1"
targetdir="$2"
# Prevent the workdir from being inside the targetdir so it isn't included in
# the update mar.
-if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+if [ "$(echo "$targetdir" | grep -c '\/$')" = 1 ]; then
# Remove the /
- targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+ targetdir=${targetdir%/}
fi
workdir="$targetdir.work"
updatemanifestv2="$workdir/updatev2.manifest"
updatemanifestv3="$workdir/updatev3.manifest"
targetfiles="updatev2.manifest updatev3.manifest"
mkdir -p "$workdir"
# Generate a list of all files in the target directory.
-pushd "$targetdir"
+pushd "$targetdir" || exit 1
if test $? -ne 0 ; then
exit 1
fi
if [ ! -f "precomplete" ]; then
if [ ! -f "Contents/Resources/precomplete" ]; then
notice "precomplete file is missing!"
exit 1
fi
fi
+declare -a files
list_files files
-popd
+popd || exit 1
# Add the type of update to the beginning of the update manifests.
-> $updatemanifestv2
-> $updatemanifestv3
+: > "$updatemanifestv2"
+: > "$updatemanifestv3"
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
-echo "type \"complete\"" >> $updatemanifestv2
-echo "type \"complete\"" >> $updatemanifestv3
+echo "type \"complete\"" >> "$updatemanifestv2"
+echo "type \"complete\"" >> "$updatemanifestv3"
notice ""
notice "Adding file add instructions to update manifests"
-num_files=${#files[*]}
-for ((i=0; $i<$num_files; i=$i+1)); do
- f="${files[$i]}"
-
+for f in "${files[@]}"
+do
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
if check_for_add_to_manifestv2 "$f"; then
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi