Bug 641212 Part 6 - Remove usage of bzip2 from the MAR builder scripts. r?rstrong, r?bhearsum draft
authorMatt Howell <mhowell@mozilla.com>
Thu, 26 Jan 2017 15:02:18 -0800
changeset 607658 f40d4bc50c3546e508771e0214401242f54391e1
parent 607657 1cba5ce0b41469b70b0effd8743505157e26aa06
child 607659 596043d866cae0bc6850a4c8a31d973e38416cd1
push id68072
push usermhowell@mozilla.com
push dateWed, 12 Jul 2017 17:32:12 +0000
reviewersrstrong, bhearsum
bugs641212
milestone56.0a1
Bug 641212 Part 6 - Remove usage of bzip2 from the MAR builder scripts. r?rstrong, r?bhearsum MozReview-Commit-ID: 7GxpXogClTy
tools/update-packaging/common.sh
tools/update-packaging/make_full_update.sh
tools/update-packaging/make_incremental_update.sh
tools/update-packaging/make_incremental_updates.py
tools/update-packaging/test/catmanifest.sh
tools/update-packaging/test/common.sh
tools/update-packaging/test/diffmar.sh
tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest
tools/update-packaging/test/from/chrome.manifest
tools/update-packaging/test/make_full_update.sh
tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest
tools/update-packaging/test/to/chrome.manifest
tools/update-packaging/unwrap_full_update.pl
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -6,17 +6,16 @@
 #
 # Code shared by update packaging scripts.
 # Author: Darin Fisher
 #
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
   echo "$*" 1>&2
 }
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -88,30 +88,27 @@ for ((i=0; $i<$num_files; i=$i+1)); do
       make_add_instruction "$f" "$updatemanifestv2" "" 1
     fi
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  cp "$targetdir/$f" "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
 then
   mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
 fi
 mar_command="$mar_command -C \"$workdir\" -c output.mar"
 eval "$mar_command $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -164,27 +164,27 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
   f="${oldfiles[$i]}"
 
   # If this file exists in the new directory as well, then check if it differs.
   if [ -f "$newdir/$f" ]; then
 
     if check_for_add_if_not_update "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      cp "$newdir/$f" "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_if_not_instruction "$f" "$updatemanifestv3"
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if check_for_forced_update "$requested_forced_updates" "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      cp "$newdir/$f" "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
       archivefiles="$archivefiles \"$f\""
       continue 1
     fi
 
     if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
       # Compute both the compressed binary diff and the compressed file, and
@@ -202,31 +202,29 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
       # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
       #   PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
       #
       # Note: patches are bzipped stashed in funsize to gain more speed
 
       # if service is not enabled then default to old behavior
       if [ -z "$MBSDIFF_HOOK" ]; then
         $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-        $BZIP2 -z9 "$workdir/$f.patch"
       else
         # if service enabled then check patch existence for retrieval
-        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"; then
           notice "file \"$f\" found in funsize, diffing skipped"
         else
           # if not found already - compute it and cache it for future use
           $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-          $BZIP2 -z9 "$workdir/$f.patch"
-          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
         fi
       fi
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      cp "$newdir/$f" "$workdir/$f"
       copy_perm "$newdir/$f" "$workdir/$f"
-      patchfile="$workdir/$f.patch.bz2"
+      patchfile="$workdir/$f.patch"
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
       if [ $patchsize -lt $fullsize ]; then
         make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
         mv -f "$patchfile" "$workdir/$f.patch"
         rm -f "$workdir/$f"
         archivefiles="$archivefiles \"$f.patch\""
@@ -257,17 +255,17 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
     if [ "$f" = "${oldfiles[j]}" ]; then
       continue 2
     fi
   done
 
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
 
-  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+  cp "$newdir/$f" "$workdir/$f"
   copy_perm "$newdir/$f" "$workdir/$f"
 
   if check_for_add_if_not_update "$f"; then
     make_add_if_not_instruction "$f" "$updatemanifestv3"
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
@@ -298,19 +296,16 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
   # If this dir doesn't exist in the new directory remove it.
   if [ ! -d "$newdir/$f" ]; then
     notice "      rmdir $f/"
     echo "rmdir \"$f/\"" >> $updatemanifestv2
     echo "rmdir \"$f/\"" >> $updatemanifestv3
   fi
 done
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
 then
   mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
 fi
 if [[ -n $MOZ_CHANNEL_ID ]]
 then
   mar_command="$mar_command -H $MOZ_CHANNEL_ID"
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -7,17 +7,16 @@ import shutil
 import sha
 from os.path import join, getsize
 from stat import *
 import re
 import sys
 import getopt
 import time
 import datetime
-import bz2
 import string
 import tempfile
 
 class PatchInfo:
     """ Represents the meta-data associated with a patch
         work_dir = working dir where files are stored for this patch
         archive_files = list of files to include in this patch
         manifestv2 = set of manifest version 2 patch instructions
@@ -107,28 +106,26 @@ class PatchInfo:
         """ Create the v2 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
         manifest_file = open(manifest_file_path, "wb")
         manifest_file.writelines("type \"partial\"\n")
         manifest_file.writelines(string.join(self.manifestv2, '\n'))
         manifest_file.writelines("\n")
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
         self.archive_files.append('"updatev2.manifest"')
 
         """ Create the v3 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
         manifest_file = open(manifest_file_path, "wb")
         manifest_file.writelines("type \"partial\"\n")
         manifest_file.writelines(string.join(self.manifestv3, '\n'))
         manifest_file.writelines("\n")
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
         self.archive_files.append('"updatev3.manifest"')
 
     def build_marfile_entry_hash(self, root_path):
         """ Iterates through the root_path, creating a MarFileEntry for each file
             and directory in that path.  Excludes any filenames in the file_exclusion_list
         """
         mar_entry_hash = {}
         filename_set = set()
@@ -194,66 +191,45 @@ def exec_shell_cmd(cmd):
 def copy_file(src_file_abs_path, dst_file_abs_path):
     """ Copies src to dst creating any parent dirs required in dst first """
     dst_file_dir=os.path.dirname(dst_file_abs_path)
     if not os.path.exists(dst_file_dir):
          os.makedirs(dst_file_dir)
     # Copy the file over
     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 
-def bzip_file(filename):
-    """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
-        assumes the path is absolute"""
-    exec_shell_cmd('bzip2 -z9 "' + filename+'"')
-    os.rename(filename+".bz2",filename)
-
-def bunzip_file(filename):
-    """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
-        doesn't matter if the filename ends in .bz2 or not"""
-    if not filename.endswith(".bz2"):
-        os.rename(filename, filename+".bz2")
-        filename=filename+".bz2"
-    exec_shell_cmd('bzip2 -d "' + filename+'"')
-
-
 def extract_mar(filename, work_dir):
     """ Extracts the marfile intot he work_dir
         assumes work_dir already exists otherwise will throw osError"""
     print "Extracting "+filename+" to "+work_dir
     saved_path = os.getcwd()
     try:
         os.chdir(work_dir)
         exec_shell_cmd("mar -x "+filename)
     finally:
         os.chdir(saved_path)
 
 def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
     """ Creates the partial patch file and manifest entry for the pair of files passed in
     """
     if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
         print 'diffing "'+from_marfile_entry.name+'\"'
-        #bunzip to/from
-        bunzip_file(from_marfile_entry.abs_path)
-        bunzip_file(to_marfile_entry.abs_path)
-
         # The patch file will be created in the working directory with the
         # name of the file in the mar + .patch
         patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
         patch_file_dir=os.path.dirname(patch_file_abs_path)
         if not os.path.exists(patch_file_dir):
             os.makedirs(patch_file_dir)
 
-        # Create bzip'd patch file
+        # Create patch file
         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
-        bzip_file(patch_file_abs_path)
 
-        # Create bzip's full file
+        # Create full file
         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
-        bzip_file(full_file_abs_path)
 
         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
             # Patch is smaller than file.  Remove the file and add patch to manifest
             os.remove(full_file_abs_path)
             file_in_manifest_name = from_marfile_entry.name+".patch"
             file_in_manifest_abspath = patch_file_abs_path
             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
         else:
@@ -299,17 +275,17 @@ def process_explicit_remove_files(dir_pa
     found in that file to the patch_info"""
 
     # Windows and linux have this file at the root of the dir
     list_file_path = os.path.join(dir_path, "removed-files")
     if not os.path.exists(list_file_path):
         list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
 
     if (os.path.exists(list_file_path)):
-        list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
+        list_file = open(list_file_path, 'r')
 
         lines = []
         for line in list_file:
             lines.append(line.strip())
         list_file.close()
 
         lines.sort(reverse=True)
         for line in lines:
@@ -426,17 +402,17 @@ def get_buildid(work_dir):
     """
     ini = '%s/application.ini' % work_dir
     if not os.path.exists(ini):
         ini = '%s/Contents/Resources/application.ini' % work_dir
         if not os.path.exists(ini):
             print 'WARNING: application.ini not found, cannot find build ID'
             return ''
 
-    file = bz2.BZ2File(ini)
+    file = open(ini, 'r')
     for line in file:
         if line.find('BuildID') == 0:
             return line.strip().split('=')[1]
     print 'WARNING: cannot find build ID in application.ini'
     return ''
 
 def decode_filename(filepath):
     """ Breaks filename/dir structure into component parts based on regex
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -4,11 +4,9 @@
 mar="$1"
 workdir="/tmp/catmanifest"
 
 rm -rf "$workdir"
 mkdir -p "$workdir"
 cp "$1" "$workdir"
 cd "$workdir"
 mar -x "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
 cat updatev2.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -7,17 +7,16 @@
 # Code shared by update packaging scripts.
 # Author: Darin Fisher
 #
 # In here to use the local common.sh to allow the full mars to have unfiltered files
 
 # -----------------------------------------------------------------------------
 # By default just assume that these tools exist on our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
 # Helper routines
 
 notice() {
   echo "$*" 1>&2
 }
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -25,27 +25,19 @@ mkdir -p "$fromdir"
 mkdir -p "$todir"
 
 cp "$1" "$fromdir"
 cp "$2" "$todir"
 
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
 ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
 ls $lsargs > files.txt
 
 echo "diffing $fromdir and $todir"
 echo "on linux shell sort and python sort return different results"
 echo "which can cause differences in the manifest files"
 diff -ru "$fromdir" "$todir"
new file mode 100644
new file mode 100644
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -89,30 +89,27 @@ for ((i=0; $i<$num_files; i=$i+1)); do
       make_add_instruction "$f" "$updatemanifestv2" "" 1
     fi
   else
     make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
   fi
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  cp "$targetdir/$f" "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
 done
 
 # Append remove instructions for any dead files.
 notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
 
 notice ""
 notice "Finished"
new file mode 100644
new file mode 100644
--- a/tools/update-packaging/unwrap_full_update.pl
+++ b/tools/update-packaging/unwrap_full_update.pl
@@ -4,64 +4,39 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 #
 # This tool unpacks a full update package generated by make_full_update.sh
 # Author: Benjamin Smedberg
 #
 
 # -----------------------------------------------------------------------------
-# By default just assume that these tools exist on our path
+# By default just assume that mar exists on our path
 
 use Getopt::Std;
 
-my ($MAR, $BZIP2, $archive, @marentries, @marfiles);
+my $MAR;
 
 if (defined($ENV{"MAR"})) {
     $MAR = $ENV{"MAR"};
 }
 else {
     $MAR = "mar";
 }
 
-if (defined($ENV{"BZIP2"})) {
-    $BZIP2 = $ENV{"BZIP2"};
-}
-else {
-    $BZIP2 = "bzip2";
-}
-
 sub print_usage
 {
     print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n";
     print "The contents of ARCHIVE will be unpacked into the current directory.\n\n";
     print "Options:\n";
     print "  -h show this help text\n";
 }
 
 my %opts;
 getopts("h", \%opts);
 
 if (defined($opts{'h'}) || scalar(@ARGV) != 1) {
     print_usage();
     exit 1;
 }
 
-$archive = $ARGV[0];
-@marentries = `"$MAR" -t "$archive"`;
-
-$? && die("Couldn't run \"$MAR\" -t");
-
-shift @marentries;
-
+my $archive = $ARGV[0];
 system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x";
-
-foreach (@marentries) {
-    tr/\n\r//d;
-    my @splits = split(/\t/,$_);
-    my $file = $splits[2];
-
-    system("mv \"$file\" \"$file.bz2\"") == 0 ||
-      die "Couldn't mv \"$file\"";
-    system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
-      die "Couldn't decompress \"$file\"";
-}
-