Bug 641212 - Part 4: Support for creating MAR files compressed using XZ; r?rstrong
I also added support to these scripts for paths containing spaces.
MozReview-Commit-ID: IIgX86jFwZ0
--- a/tools/update-packaging/Makefile.in
+++ b/tools/update-packaging/Makefile.in
@@ -61,19 +61,25 @@ ifeq ($(OS_TARGET), WINNT)
endif
MAR=$(MAR_BIN) \
$(srcdir)/make_full_update.sh \
'$(DIST)/$(COMPLETE_MAR)' \
'$(PACKAGE_DIR)'
ifdef MOZ_SIGN_CMD
$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)'
endif
+ xz --compress --extreme --format=xz --check=crc64 '$(DIST)/$(COMPLETE_MAR)'
+ mv '$(DIST)/$(COMPLETE_MAR).xz' '$(DIST)/$(COMPLETE_MAR)'
partial-patch:: $(dir-stage)
MAR=$(MAR_BIN) \
MBSDIFF=$(MBSDIFF_BIN) \
$(srcdir)/make_incremental_update.sh \
'$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' \
'$(SRC_BUILD)' \
'$(DST_BUILD)'
ifdef MOZ_SIGN_CMD
$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
endif
+ xz --compress --extreme --format=xz --check=crc64 \
+ '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
+ mv '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar.xz' \
+ '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -5,16 +5,17 @@
#
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
+XZ=${XZ:-xz}
MAR=${MAR:-mar}
BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff}
# -----------------------------------------------------------------------------
# Helper routines
notice() {
@@ -55,25 +56,25 @@ make_add_instruction() {
fi
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " add-if \"$testdir\" \"$f\""
- echo "add-if \"$testdir\" \"$f\"" >> $filev2
- if [ ! $filev3 = "" ]; then
- echo "add-if \"$testdir\" \"$f\"" >> $filev3
+ echo "add-if \"$testdir\" \"$f\"" >> "$filev2"
+ if [ ! "$filev3" = "" ]; then
+ echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
fi
else
notice " add \"$f\"$forced"
- echo "add \"$f\"" >> $filev2
- if [ ! $filev3 = "" ]; then
- echo "add \"$f\"" >> $filev3
+ echo "add \"$f\"" >> "$filev2"
+ if [ ! "$filev3" = "" ]; then
+ echo "add \"$f\"" >> "$filev3"
fi
fi
}
check_for_add_if_not_update() {
add_if_not_file_chk="$1"
if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
@@ -96,36 +97,36 @@ check_for_add_to_manifestv2() {
return 1;
}
make_add_if_not_instruction() {
f="$1"
filev3="$2"
notice " add-if-not \"$f\" \"$f\""
- echo "add-if-not \"$f\" \"$f\"" >> $filev3
+ echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
}
make_patch_instruction() {
f="$1"
filev2="$2"
filev3="$3"
is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
if [ $is_extension = "1" ]; then
# Use the subdirectory of the extensions folder as the file to test
# before performing this add instruction.
testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
notice " patch-if \"$testdir\" \"$f.patch\" \"$f\""
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
- echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
+ echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2"
+ echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
else
notice " patch \"$f.patch\" \"$f\""
- echo "patch \"$f.patch\" \"$f\"" >> $filev2
- echo "patch \"$f.patch\" \"$f\"" >> $filev3
+ echo "patch \"$f.patch\" \"$f\"" >> "$filev2"
+ echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
fi
}
append_remove_instructions() {
dir="$1"
filev2="$2"
filev3="$3"
@@ -144,28 +145,28 @@ append_remove_instructions() {
# Trim whitespace
f=$(echo $f)
# Exclude blank lines.
if [ -n "$f" ]; then
# Exclude comments
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
notice " rmdir \"$f\""
- echo "rmdir \"$f\"" >> $filev2
- echo "rmdir \"$f\"" >> $filev3
+ echo "rmdir \"$f\"" >> "$filev2"
+ echo "rmdir \"$f\"" >> "$filev3"
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
# Remove the *
f=$(echo "$f" | sed -e 's:\*$::')
notice " rmrfdir \"$f\""
- echo "rmrfdir \"$f\"" >> $filev2
- echo "rmrfdir \"$f\"" >> $filev3
+ echo "rmrfdir \"$f\"" >> "$filev2"
+ echo "rmrfdir \"$f\"" >> "$filev3"
else
notice " remove \"$f\""
- echo "remove \"$f\"" >> $filev2
- echo "remove \"$f\"" >> $filev3
+ echo "remove \"$f\"" >> "$filev2"
+ echo "remove \"$f\"" >> "$filev3"
fi
fi
fi
done
fi
}
# List all files in the current directory, stripping leading "./"
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -62,23 +62,23 @@ if [ ! -f "precomplete" ]; then
fi
fi
list_files files
popd
# Add the type of update to the beginning of the update manifests.
-> $updatemanifestv2
-> $updatemanifestv3
+> "$updatemanifestv2"
+> "$updatemanifestv3"
notice ""
notice "Adding type instruction to update manifests"
notice " type complete"
-echo "type \"complete\"" >> $updatemanifestv2
-echo "type \"complete\"" >> $updatemanifestv3
+echo "type \"complete\"" >> "$updatemanifestv2"
+echo "type \"complete\"" >> "$updatemanifestv3"
notice ""
notice "Adding file add instructions to update manifests"
num_files=${#files[*]}
for ((i=0; $i<$num_files; i=$i+1)); do
f="${files[$i]}"
@@ -88,30 +88,25 @@ for ((i=0; $i<$num_files; i=$i+1)); do
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
- $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
- copy_perm "$targetdir/$f" "$workdir/$f"
-
+ cp -p "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
# cleanup
rm -fr "$workdir"
notice ""
notice "Finished"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -142,91 +142,87 @@ fi
list_dirs newdirs
list_files newfiles
popd
# Add the type of update to the beginning of the update manifests.
notice ""
notice "Adding type instruction to update manifests"
-> $updatemanifestv2
-> $updatemanifestv3
+> "$updatemanifestv2"
+> "$updatemanifestv3"
notice " type partial"
-echo "type \"partial\"" >> $updatemanifestv2
-echo "type \"partial\"" >> $updatemanifestv3
+echo "type \"partial\"" >> "$updatemanifestv2"
+echo "type \"partial\"" >> "$updatemanifestv3"
notice ""
notice "Adding file patch and add instructions to update manifests"
num_oldfiles=${#oldfiles[*]}
remove_array=
num_removes=0
for ((i=0; $i<$num_oldfiles; i=$i+1)); do
f="${oldfiles[$i]}"
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
- mkdir -p `dirname "$workdir/$f"`
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
- copy_perm "$newdir/$f" "$workdir/$f"
+ mkdir -p "`dirname "$workdir/$f"`"
+ cp -p "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
continue 1
fi
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
- mkdir -p `dirname "$workdir/$f"`
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
- copy_perm "$newdir/$f" "$workdir/$f"
+ mkdir -p "`dirname "$workdir/$f"`"
+ cp -p "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
continue 1
fi
if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
# Compute both the compressed binary diff and the compressed file, and
# compare the sizes. Then choose the smaller of the two to package.
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
notice "diffing \"$f\""
+ patchfile="$workdir/$f.patch"
# MBSDIFF_HOOK represents the communication interface with funsize and,
# if enabled, caches the intermediate patches for future use and
# compute avoidance
#
# An example of MBSDIFF_HOOK env variable could look like this:
# export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
# where myscript.sh has the following usage:
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
#
# Note: patches are bzipped stashed in funsize to gain more speed
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $BZIP2 -z9 "$workdir/$f.patch"
+ $MBSDIFF "$olddir/$f" "$newdir/$f" "$patchfile"
else
# if service enabled then check patch existence for retrieval
- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+ if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$patchfile"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
- $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $BZIP2 -z9 "$workdir/$f.patch"
- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+ $MBSDIFF "$olddir/$f" "$newdir/$f" "$patchfile"
+ $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$patchfile"
fi
fi
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
- copy_perm "$newdir/$f" "$workdir/$f"
- patchfile="$workdir/$f.patch.bz2"
+
+ cp -p "$newdir/$f" "$workdir/$f"
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
archivefiles="$archivefiles \"$f.patch\""
@@ -256,61 +252,56 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
for ((j=0; $j<$num_oldfiles; j=$j+1)); do
if [ "$f" = "${oldfiles[j]}" ]; then
continue 2
fi
done
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
-
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
- copy_perm "$newdir/$f" "$workdir/$f"
+ cp -p "$targetdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
archivefiles="$archivefiles \"$f\""
done
notice ""
notice "Adding file remove instructions to update manifests"
for ((i=0; $i<$num_removes; i=$i+1)); do
f="${remove_array[$i]}"
notice " remove \"$f\""
- echo "remove \"$f\"" >> $updatemanifestv2
- echo "remove \"$f\"" >> $updatemanifestv3
+ echo "remove \"$f\"" >> "$updatemanifestv2"
+ echo "remove \"$f\"" >> "$updatemanifestv3"
done
# Add remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
notice ""
notice "Adding directory remove instructions for directories that no longer exist"
num_olddirs=${#olddirs[*]}
for ((i=0; $i<$num_olddirs; i=$i+1)); do
f="${olddirs[$i]}"
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/"
- echo "rmdir \"$f/\"" >> $updatemanifestv2
- echo "rmdir \"$f/\"" >> $updatemanifestv3
+ echo "rmdir \"$f/\"" >> "$updatemanifestv2"
+ echo "rmdir \"$f/\"" >> "$updatemanifestv3"
fi
done
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
mar_command="$MAR"
if [[ -n $MOZ_PRODUCT_VERSION ]]
then
mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
fi
if [[ -n $MOZ_CHANNEL_ID ]]
then
mar_command="$mar_command -H $MOZ_CHANNEL_ID"