Bug 641212 Part 6 - Remove usage of bzip2 from the MAR builder scripts. r?rstrong, r?bhearsum
MozReview-Commit-ID: 7GxpXogClTy
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -6,17 +6,16 @@
#
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff}
# -----------------------------------------------------------------------------
# Helper routines
notice() {
echo "$*" 1>&2
}
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -88,30 +88,27 @@ for ((i=0; $i<$num_files; i=$i+1)); do
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
- $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+ cp "$targetdir/$f" "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
mar_command="$MAR"
if [[ -n $MOZ_PRODUCT_VERSION ]]
then
mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
fi
mar_command="$mar_command -C \"$workdir\" -c output.mar"
eval "$mar_command $targetfiles"
mv -f "$workdir/output.mar" "$archive"
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -164,27 +164,27 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
f="${oldfiles[$i]}"
# If this file exists in the new directory as well, then check if it differs.
if [ -f "$newdir/$f" ]; then
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+ cp "$newdir/$f" "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
continue 1
fi
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+ cp "$newdir/$f" "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
continue 1
fi
if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
# Compute both the compressed binary diff and the compressed file, and
@@ -202,31 +202,29 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); d
# myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \
# PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL
#
# Note: patches are bzipped stashed in funsize to gain more speed
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $BZIP2 -z9 "$workdir/$f.patch"
else
# if service enabled then check patch existence for retrieval
- if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+ if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"; then
notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
- $BZIP2 -z9 "$workdir/$f.patch"
- $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+ $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
fi
fi
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+ cp "$newdir/$f" "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
- patchfile="$workdir/$f.patch.bz2"
+ patchfile="$workdir/$f.patch"
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
if [ $patchsize -lt $fullsize ]; then
make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
mv -f "$patchfile" "$workdir/$f.patch"
rm -f "$workdir/$f"
archivefiles="$archivefiles \"$f.patch\""
@@ -257,17 +255,17 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); d
if [ "$f" = "${oldfiles[j]}" ]; then
continue 2
fi
done
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
- $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+ cp "$newdir/$f" "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then
make_add_if_not_instruction "$f" "$updatemanifestv3"
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
@@ -298,19 +296,16 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
# If this dir doesn't exist in the new directory remove it.
if [ ! -d "$newdir/$f" ]; then
notice " rmdir $f/"
echo "rmdir \"$f/\"" >> $updatemanifestv2
echo "rmdir \"$f/\"" >> $updatemanifestv3
fi
done
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
mar_command="$MAR"
if [[ -n $MOZ_PRODUCT_VERSION ]]
then
mar_command="$mar_command -V $MOZ_PRODUCT_VERSION"
fi
if [[ -n $MOZ_CHANNEL_ID ]]
then
mar_command="$mar_command -H $MOZ_CHANNEL_ID"
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -7,17 +7,16 @@ import shutil
import sha
from os.path import join, getsize
from stat import *
import re
import sys
import getopt
import time
import datetime
-import bz2
import string
import tempfile
class PatchInfo:
""" Represents the meta-data associated with a patch
work_dir = working dir where files are stored for this patch
archive_files = list of files to include in this patch
manifestv2 = set of manifest version 2 patch instructions
@@ -107,28 +106,26 @@ class PatchInfo:
""" Create the v2 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv2, '\n'))
manifest_file.writelines("\n")
manifest_file.close()
- bzip_file(manifest_file_path)
self.archive_files.append('"updatev2.manifest"')
""" Create the v3 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv3, '\n'))
manifest_file.writelines("\n")
manifest_file.close()
- bzip_file(manifest_file_path)
self.archive_files.append('"updatev3.manifest"')
def build_marfile_entry_hash(self, root_path):
""" Iterates through the root_path, creating a MarFileEntry for each file
and directory in that path. Excludes any filenames in the file_exclusion_list
"""
mar_entry_hash = {}
filename_set = set()
@@ -194,66 +191,45 @@ def exec_shell_cmd(cmd):
def copy_file(src_file_abs_path, dst_file_abs_path):
""" Copies src to dst creating any parent dirs required in dst first """
dst_file_dir=os.path.dirname(dst_file_abs_path)
if not os.path.exists(dst_file_dir):
os.makedirs(dst_file_dir)
# Copy the file over
shutil.copy2(src_file_abs_path, dst_file_abs_path)
-def bzip_file(filename):
- """ Bzip's the file in place. The original file is replaced with a bzip'd version of itself
- assumes the path is absolute"""
- exec_shell_cmd('bzip2 -z9 "' + filename+'"')
- os.rename(filename+".bz2",filename)
-
-def bunzip_file(filename):
- """ Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself.
- doesn't matter if the filename ends in .bz2 or not"""
- if not filename.endswith(".bz2"):
- os.rename(filename, filename+".bz2")
- filename=filename+".bz2"
- exec_shell_cmd('bzip2 -d "' + filename+'"')
-
-
def extract_mar(filename, work_dir):
""" Extracts the marfile intot he work_dir
assumes work_dir already exists otherwise will throw osError"""
print "Extracting "+filename+" to "+work_dir
saved_path = os.getcwd()
try:
os.chdir(work_dir)
exec_shell_cmd("mar -x "+filename)
finally:
os.chdir(saved_path)
def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
""" Creates the partial patch file and manifest entry for the pair of files passed in
"""
if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
print 'diffing "'+from_marfile_entry.name+'\"'
- #bunzip to/from
- bunzip_file(from_marfile_entry.abs_path)
- bunzip_file(to_marfile_entry.abs_path)
-
# The patch file will be created in the working directory with the
# name of the file in the mar + .patch
patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
patch_file_dir=os.path.dirname(patch_file_abs_path)
if not os.path.exists(patch_file_dir):
os.makedirs(patch_file_dir)
- # Create bzip'd patch file
+ # Create patch file
exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
- bzip_file(patch_file_abs_path)
- # Create bzip's full file
+ # Create full file
full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
- bzip_file(full_file_abs_path)
if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
# Patch is smaller than file. Remove the file and add patch to manifest
os.remove(full_file_abs_path)
file_in_manifest_name = from_marfile_entry.name+".patch"
file_in_manifest_abspath = patch_file_abs_path
patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
else:
@@ -299,17 +275,17 @@ def process_explicit_remove_files(dir_pa
found in that file to the patch_info"""
# Windows and linux have this file at the root of the dir
list_file_path = os.path.join(dir_path, "removed-files")
if not os.path.exists(list_file_path):
list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
if (os.path.exists(list_file_path)):
- list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
+ list_file = open(list_file_path, 'r')
lines = []
for line in list_file:
lines.append(line.strip())
list_file.close()
lines.sort(reverse=True)
for line in lines:
@@ -426,17 +402,17 @@ def get_buildid(work_dir):
"""
ini = '%s/application.ini' % work_dir
if not os.path.exists(ini):
ini = '%s/Contents/Resources/application.ini' % work_dir
if not os.path.exists(ini):
print 'WARNING: application.ini not found, cannot find build ID'
return ''
- file = bz2.BZ2File(ini)
+ file = open(ini, 'r')
for line in file:
if line.find('BuildID') == 0:
return line.strip().split('=')[1]
print 'WARNING: cannot find build ID in application.ini'
return ''
def decode_filename(filepath):
""" Breaks filename/dir structure into component parts based on regex
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -4,11 +4,9 @@
mar="$1"
workdir="/tmp/catmanifest"
rm -rf "$workdir"
mkdir -p "$workdir"
cp "$1" "$workdir"
cd "$workdir"
mar -x "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
cat updatev2.manifest
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -7,17 +7,16 @@
# Code shared by update packaging scripts.
# Author: Darin Fisher
#
# In here to use the local common.sh to allow the full mars to have unfiltered files
# -----------------------------------------------------------------------------
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
MBSDIFF=${MBSDIFF:-mbsdiff}
# -----------------------------------------------------------------------------
# Helper routines
notice() {
echo "$*" 1>&2
}
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -25,27 +25,19 @@ mkdir -p "$fromdir"
mkdir -p "$todir"
cp "$1" "$fromdir"
cp "$2" "$todir"
cd "$fromdir"
mar -x "$1"
rm "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
ls $lsargs > files.txt
cd "$todir"
mar -x "$2"
rm "$2"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
ls $lsargs > files.txt
echo "diffing $fromdir and $todir"
echo "on linux shell sort and python sort return different results"
echo "which can cause differences in the manifest files"
diff -ru "$fromdir" "$todir"
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -89,30 +89,27 @@ for ((i=0; $i<$num_files; i=$i+1)); do
make_add_instruction "$f" "$updatemanifestv2" "" 1
fi
else
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
fi
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
- $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+ cp "$targetdir/$f" "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
done
# Append remove instructions for any dead files.
notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
-
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
mv -f "$workdir/output.mar" "$archive"
# cleanup
rm -fr "$workdir"
notice ""
notice "Finished"
--- a/tools/update-packaging/unwrap_full_update.pl
+++ b/tools/update-packaging/unwrap_full_update.pl
@@ -4,64 +4,39 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool unpacks a full update package generated by make_full_update.sh
# Author: Benjamin Smedberg
#
# -----------------------------------------------------------------------------
-# By default just assume that these tools exist on our path
+# By default just assume that mar exists on our path
use Getopt::Std;
-my ($MAR, $BZIP2, $archive, @marentries, @marfiles);
+my $MAR;
if (defined($ENV{"MAR"})) {
$MAR = $ENV{"MAR"};
}
else {
$MAR = "mar";
}
-if (defined($ENV{"BZIP2"})) {
- $BZIP2 = $ENV{"BZIP2"};
-}
-else {
- $BZIP2 = "bzip2";
-}
-
sub print_usage
{
print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n";
print "The contents of ARCHIVE will be unpacked into the current directory.\n\n";
print "Options:\n";
print " -h show this help text\n";
}
my %opts;
getopts("h", \%opts);
if (defined($opts{'h'}) || scalar(@ARGV) != 1) {
print_usage();
exit 1;
}
-$archive = $ARGV[0];
-@marentries = `"$MAR" -t "$archive"`;
-
-$? && die("Couldn't run \"$MAR\" -t");
-
-shift @marentries;
-
+my $archive = $ARGV[0];
system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x";
-
-foreach (@marentries) {
- tr/\n\r//d;
- my @splits = split(/\t/,$_);
- my $file = $splits[2];
-
- system("mv \"$file\" \"$file.bz2\"") == 0 ||
- die "Couldn't mv \"$file\"";
- system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
- die "Couldn't decompress \"$file\"";
-}
-