Bug 641212 - Part 4: Support for creating MAR files compressed using XZ; r?rstrong draft
authorMatt Howell <mhowell@mozilla.com>
Tue, 15 Nov 2016 13:41:15 -0800
changeset 445553 901bb025170cf4a9b5610338e69096a427948662
parent 445552 86f98e76522d564a93b470315504c7febbdf7e96
child 538558 cd6d782716e05114f0a44604ea867ff95ee2b9a8
push id37551
push usermhowell@mozilla.com
push dateTue, 29 Nov 2016 23:10:16 +0000
reviewersrstrong
bugs641212
milestone53.0a1
Bug 641212 - Part 4: Support for creating MAR files compressed using XZ; r?rstrong I also added support to these scripts for paths containing spaces. MozReview-Commit-ID: IIgX86jFwZ0
tools/update-packaging/Makefile.in
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
--- a/tools/update-packaging/Makefile.in
+++ b/tools/update-packaging/Makefile.in
@@ -61,19 +61,25 @@ ifeq ($(OS_TARGET), WINNT)
 endif
 	MAR=$(MAR_BIN) \
 	  $(srcdir)/make_full_update.sh \
 	  '$(DIST)/$(COMPLETE_MAR)' \
 	  '$(PACKAGE_DIR)'
 ifdef MOZ_SIGN_CMD
 	$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)'
 endif
+	xz --compress --extreme --format=xz --check=crc64 '$(DIST)/$(COMPLETE_MAR)'
+	mv '$(DIST)/$(COMPLETE_MAR).xz' '$(DIST)/$(COMPLETE_MAR)'
 
 partial-patch:: $(dir-stage)
 	MAR=$(MAR_BIN) \
 	MBSDIFF=$(MBSDIFF_BIN) \
 	  $(srcdir)/make_incremental_update.sh \
 	  '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar' \
 	  '$(SRC_BUILD)' \
 	  '$(DST_BUILD)'
 ifdef MOZ_SIGN_CMD
 	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
 endif
+	xz --compress --extreme --format=xz --check=crc64 \
+    '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
+	mv '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar.xz' \
+    '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -5,16 +5,17 @@
 
 #
 # Code shared by update packaging scripts.
 # Author: Darin Fisher
 #
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
+XZ=${XZ:-xz}
 MAR=${MAR:-mar}
 BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
@@ -55,25 +56,25 @@ make_add_instruction() {
   fi
 
   is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
     notice "     add-if \"$testdir\" \"$f\""
-    echo "add-if \"$testdir\" \"$f\"" >> $filev2
-    if [ ! $filev3 = "" ]; then
-      echo "add-if \"$testdir\" \"$f\"" >> $filev3
+    echo "add-if \"$testdir\" \"$f\"" >> "$filev2"
+    if [ ! "$filev3" = "" ]; then
+      echo "add-if \"$testdir\" \"$f\"" >> "$filev3"
     fi
   else
     notice "        add \"$f\"$forced"
-    echo "add \"$f\"" >> $filev2
-    if [ ! $filev3 = "" ]; then
-      echo "add \"$f\"" >> $filev3
+    echo "add \"$f\"" >> "$filev2"
+    if [ ! "$filev3" = "" ]; then
+      echo "add \"$f\"" >> "$filev3"
     fi
   fi
 }
 
 check_for_add_if_not_update() {
   add_if_not_file_chk="$1"
 
   if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \
@@ -96,36 +97,36 @@ check_for_add_to_manifestv2() {
   return 1;
 }
 
 make_add_if_not_instruction() {
   f="$1"
   filev3="$2"
 
   notice " add-if-not \"$f\" \"$f\""
-  echo "add-if-not \"$f\" \"$f\"" >> $filev3
+  echo "add-if-not \"$f\" \"$f\"" >> "$filev3"
 }
 
 make_patch_instruction() {
   f="$1"
   filev2="$2"
   filev3="$3"
 
   is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/')
     notice "   patch-if \"$testdir\" \"$f.patch\" \"$f\""
-    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2
-    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3
+    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev2"
+    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3"
   else
     notice "      patch \"$f.patch\" \"$f\""
-    echo "patch \"$f.patch\" \"$f\"" >> $filev2
-    echo "patch \"$f.patch\" \"$f\"" >> $filev3
+    echo "patch \"$f.patch\" \"$f\"" >> "$filev2"
+    echo "patch \"$f.patch\" \"$f\"" >> "$filev3"
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
   filev2="$2"
   filev3="$3"
 
@@ -144,28 +145,28 @@ append_remove_instructions() {
       # Trim whitespace
       f=$(echo $f)
       # Exclude blank lines.
       if [ -n "$f" ]; then
         # Exclude comments
         if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
           if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
             notice "      rmdir \"$f\""
-            echo "rmdir \"$f\"" >> $filev2
-            echo "rmdir \"$f\"" >> $filev3
+            echo "rmdir \"$f\"" >> "$filev2"
+            echo "rmdir \"$f\"" >> "$filev3"
           elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
             # Remove the *
             f=$(echo "$f" | sed -e 's:\*$::')
             notice "    rmrfdir \"$f\""
-            echo "rmrfdir \"$f\"" >> $filev2
-            echo "rmrfdir \"$f\"" >> $filev3
+            echo "rmrfdir \"$f\"" >> "$filev2"
+            echo "rmrfdir \"$f\"" >> "$filev3"
           else
             notice "     remove \"$f\""
-            echo "remove \"$f\"" >> $filev2
-            echo "remove \"$f\"" >> $filev3
+            echo "remove \"$f\"" >> "$filev2"
+            echo "remove \"$f\"" >> "$filev3"
           fi
         fi
       fi
     done
   fi
 }
 
 # List all files in the current directory, stripping leading "./"
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -62,23 +62,23 @@ if [ ! -f "precomplete" ]; then
   fi
 fi
 
 list_files files
 
 popd
 
 # Add the type of update to the beginning of the update manifests.
-> $updatemanifestv2
-> $updatemanifestv3
+> "$updatemanifestv2"
+> "$updatemanifestv3"
 notice ""
 notice "Adding type instruction to update manifests"
 notice "       type complete"
-echo "type \"complete\"" >> $updatemanifestv2
-echo "type \"complete\"" >> $updatemanifestv3
+echo "type \"complete\"" >> "$updatemanifestv2"
+echo "type \"complete\"" >> "$updatemanifestv3"
 
 notice ""
 notice "Adding file add instructions to update manifests"
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
@@ -88,30 +88,25 @@ for ((i=0; $i<$num_files; i=$i+1)); do
       make_add_instruction "$f" "$updatemanifestv2" "" 1
     fi
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
-  copy_perm "$targetdir/$f" "$workdir/$f"
-
+  cp -p "$targetdir/$f" "$workdir/$f"
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
 
 notice ""
 notice "Finished"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -142,91 +142,87 @@ fi
 list_dirs newdirs
 list_files newfiles
 
 popd
 
 # Add the type of update to the beginning of the update manifests.
 notice ""
 notice "Adding type instruction to update manifests"
-> $updatemanifestv2
-> $updatemanifestv3
+> "$updatemanifestv2"
+> "$updatemanifestv3"
 notice "       type partial"
-echo "type \"partial\"" >> $updatemanifestv2
-echo "type \"partial\"" >> $updatemanifestv3
+echo "type \"partial\"" >> "$updatemanifestv2"
+echo "type \"partial\"" >> "$updatemanifestv3"
 
 notice ""
 notice "Adding file patch and add instructions to update manifests"
 
 num_oldfiles=${#oldfiles[*]}
 remove_array=
 num_removes=0
 
 for ((i=0; $i<$num_oldfiles; i=$i+1)); do
   f="${oldfiles[$i]}"
 
   # If this file exists in the new directory as well, then check if it differs.
   if [ -f "$newdir/$f" ]; then
 
     if check_for_add_if_not_update "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
-      mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
+      mkdir -p "`dirname "$workdir/$f"`"
+      cp -p "$newdir/$f" "$workdir/$f"
       make_add_if_not_instruction "$f" "$updatemanifestv3"
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
-      mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
+      mkdir -p "`dirname "$workdir/$f"`"
+      cp -p "$newdir/$f" "$workdir/$f"
       make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
       # compare the sizes.  Then choose the smaller of the two to package.
       dir=$(dirname "$workdir/$f")
       mkdir -p "$dir"
       notice "diffing \"$f\""
+      patchfile="$workdir/$f.patch"
       # MBSDIFF_HOOK represents the communication interface with funsize and,
       # if enabled, caches the intermediate patches for future use and
       # compute avoidance
       #
       # An example of MBSDIFF_HOOK env variable could look like this:
       # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user"
       # where myscript.sh has the following usage:
       # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
       #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
       #
       # Note: patches are bzipped stashed in funsize to gain more speed
 
       # if service is not enabled then default to old behavior
       if [ -z "$MBSDIFF_HOOK" ]; then
-        $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-        $BZIP2 -z9 "$workdir/$f.patch"
+        $MBSDIFF "$olddir/$f" "$newdir/$f" "$patchfile"
       else
         # if service enabled then check patch existence for retrieval
-        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$patchfile"; then
           notice "file \"$f\" found in funsize, diffing skipped"
         else
           # if not found already - compute it and cache it for future use
-          $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-          $BZIP2 -z9 "$workdir/$f.patch"
-          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          $MBSDIFF "$olddir/$f" "$newdir/$f" "$patchfile"
+          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$patchfile"
         fi
       fi
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-      copy_perm "$newdir/$f" "$workdir/$f"
-      patchfile="$workdir/$f.patch.bz2"
+
+      cp -p "$newdir/$f" "$workdir/$f"
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
       if [ $patchsize -lt $fullsize ]; then
         make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
@@ -256,61 +252,56 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
   for ((j=0; $j<$num_oldfiles; j=$j+1)); do
     if [ "$f" = "${oldfiles[j]}" ]; then
       continue 2
     fi
   done
 
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
-
-  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
-  copy_perm "$newdir/$f" "$workdir/$f"
+  cp -p "$targetdir/$f" "$workdir/$f"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
 
   archivefiles="$archivefiles \"$f\""
 done
 
 notice ""
 notice "Adding file remove instructions to update manifests"
 for ((i=0; $i<$num_removes; i=$i+1)); do
   f="${remove_array[$i]}"
   notice "     remove \"$f\""
-  echo "remove \"$f\"" >> $updatemanifestv2
-  echo "remove \"$f\"" >> $updatemanifestv3
+  echo "remove \"$f\"" >> "$updatemanifestv2"
+  echo "remove \"$f\"" >> "$updatemanifestv3"
 done
 
 # Add remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
 
 notice ""
 notice "Adding directory remove instructions for directories that no longer exist"
 num_olddirs=${#olddirs[*]}
 
 for ((i=0; $i<$num_olddirs; i=$i+1)); do
   f="${olddirs[$i]}"
   # If this dir doesn't exist in the new directory remove it.
   if [ ! -d "$newdir/$f" ]; then
     notice "      rmdir $f/"
-    echo "rmdir \"$f/\"" >> $updatemanifestv2
-    echo "rmdir \"$f/\"" >> $updatemanifestv3
+    echo "rmdir \"$f/\"" >> "$updatemanifestv2"
+    echo "rmdir \"$f/\"" >> "$updatemanifestv3"
   fi
 done
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
 then
   mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
 fi
 if [[ -n $MOZ_CHANNEL_ID ]]
 then
   mar_command="$mar_command -H $MOZ_CHANNEL_ID"