Bug 641212 - Part 1: Enable the updater to extract MAR files compressed using XZ; r?rstrong draft
authorMatt Howell <mhowell@mozilla.com>
Tue, 08 Nov 2016 10:28:34 -0800
changeset 445550 7b4c14a7fe139f039e9387f798fcc73a802320a7
parent 445465 8d8846f63b74eb930e48b410730ae088e9bdbee8
child 445551 f18311a53ea078256a9be5bed24bb49897ec928e
push id37551
push usermhowell@mozilla.com
push dateTue, 29 Nov 2016 23:10:16 +0000
reviewersrstrong
bugs641212
milestone53.0a1
Bug 641212 - Part 1: Enable the updater to extract MAR files compressed using XZ; r?rstrong This patch uses the existing xz-embedded library in the tree to extract MAR files compressed using XZ. The entire MAR file is now compressed, as opposed to the previous method of compressing each individual file within the archive. This gets slightly better compression ratios than just replacing BZ2 with XZ for the individual files. This means that individual files within the (compressed) MAR are now uncompressed; continuing to use BZ2 on them would be wasteful. So support was also added for extracting uncompressed files from a MAR. Later parts will add support for creating these kinds of MARs, and will also add tests. Note that support for extracting the previous MAR compression method was not removed and still works. MozReview-Commit-ID: I41y8gRdA8u
config/external/moz.build
modules/xz-embedded/moz.build
toolkit/mozapps/update/common/errors.h
toolkit/mozapps/update/updater/archivereader.cpp
toolkit/mozapps/update/updater/archivereader.h
toolkit/mozapps/update/updater/updater-common.build
toolkit/mozapps/update/updater/updater.cpp
--- a/config/external/moz.build
+++ b/config/external/moz.build
@@ -56,12 +56,12 @@ external_dirs += [
     'media/libopus',
     'media/libtheora',
     'media/libspeex_resampler',
     'media/libstagefright',
     'media/libsoundtouch',
     'media/psshparser'
 ]
 
-if CONFIG['MOZ_LINKER']:
+if CONFIG['MOZ_LINKER'] or CONFIG['MOZ_UPDATER']:
     external_dirs += ['modules/xz-embedded']
 
 DIRS += ['../../' + i for i in external_dirs]
--- a/modules/xz-embedded/moz.build
+++ b/modules/xz-embedded/moz.build
@@ -25,9 +25,13 @@ if CONFIG['TARGET_CPU'].startswith('arm'
     else:
         DEFINES['XZ_DEC_ARM'] = 1
 elif '86' in CONFIG['TARGET_CPU']:
     # Accept x86, x86_64, i386, i686, etc.
     DEFINES['XZ_DEC_X86'] = 1
 
 DEFINES['XZ_USE_CRC64'] = 1
 
+FORCE_STATIC_LIB = True
+if CONFIG['OS_ARCH'] == 'WINNT':
+    USE_STATIC_LIBS = True
+
 Library('xz-embedded')
--- a/toolkit/mozapps/update/common/errors.h
+++ b/toolkit/mozapps/update/common/errors.h
@@ -66,16 +66,18 @@
 #define DELETE_ERROR_EXPECTED_FILE 47
 #define RENAME_ERROR_EXPECTED_FILE 48
 
 // Error codes 24-33 and 49-51 are for the Windows maintenance service.
 #define SERVICE_COULD_NOT_COPY_UPDATER 49
 #define SERVICE_STILL_APPLYING_TERMINATED 50
 #define SERVICE_STILL_APPLYING_NO_EXIT_CODE 51
 
+#define UNEXPECTED_XZ_ERROR 52
+
 #define WRITE_ERROR_FILE_COPY 61
 #define WRITE_ERROR_DELETE_FILE 62
 #define WRITE_ERROR_OPEN_PATCH_FILE 63
 #define WRITE_ERROR_PATCH_FILE 64
 #define WRITE_ERROR_APPLY_DIR_PATH 65
 #define WRITE_ERROR_CALLBACK_PATH 66
 #define WRITE_ERROR_FILE_ACCESS_DENIED 67
 #define WRITE_ERROR_DIR_ACCESS_DENIED 68
--- a/toolkit/mozapps/update/updater/archivereader.cpp
+++ b/toolkit/mozapps/update/updater/archivereader.cpp
@@ -3,18 +3,20 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include <string.h>
 #include <stdlib.h>
 #include <fcntl.h>
 #include "bzlib.h"
+#include "xz.h"
 #include "archivereader.h"
 #include "errors.h"
+#include "updatedefines.h"
 #ifdef XP_WIN
 #include "nsAlgorithm.h" // Needed by nsVersionComparator.cpp
 #include "updatehelper.h"
 #endif
 
 // These are generated at compile time based on the DER file for the channel
 // being used
 #ifdef MOZ_VERIFY_MAR_SIGNATURE
@@ -31,16 +33,19 @@
 #undef UPDATER_NO_STRING_GLUE_STL
 
 #if defined(XP_UNIX)
 # include <sys/types.h>
 #elif defined(XP_WIN)
 # include <io.h>
 #endif
 
+const unsigned char BZ2_MAGIC[] = { 'B', 'Z', 'h' };
+const unsigned char XZ_MAGIC[] = { 0xFD, '7', 'z', 'X', 'Z', 0x00 };
+
 static int inbuf_size  = 262144;
 static int outbuf_size = 262144;
 static char *inbuf  = nullptr;
 static char *outbuf = nullptr;
 
 /**
  * Performs a verification on the opened MAR file with the passed in
  * certificate name ID and type ID.
@@ -63,19 +68,19 @@ VerifyLoadedCert(MarFile *archive, const
     return CERT_VERIFY_ERROR;
   }
 #endif
 
   return OK;
 }
 
 /**
- * Performs a verification on the opened MAR file.  Both the primary and backup 
- * keys stored are stored in the current process and at least the primary key 
- * will be tried.  Success will be returned as long as one of the two 
+ * Performs a verification on the opened MAR file.  Both the primary and backup
+ * keys stored are stored in the current process and at least the primary key
+ * will be tried.  Success will be returned as long as one of the two
  * signatures verify.
  *
  * @return OK on success
 */
 int
 ArchiveReader::VerifySignature()
 {
   if (!mArchive) {
@@ -94,56 +99,56 @@ ArchiveReader::VerifySignature()
   }
 #endif
   return rv;
 #endif
 }
 
 /**
  * Verifies that the MAR file matches the current product, channel, and version
- * 
+ *
  * @param MARChannelID   The MAR channel name to use, only updates from MARs
  *                       with a matching MAR channel name will succeed.
  *                       If an empty string is passed, no check will be done
  *                       for the channel name in the product information block.
  *                       If a comma separated list of values is passed then
  *                       one value must match.
  * @param appVersion     The application version to use, only MARs with an
  *                       application version >= to appVersion will be applied.
  * @return OK on success
- *         COULD_NOT_READ_PRODUCT_INFO_BLOCK if the product info block 
+ *         COULD_NOT_READ_PRODUCT_INFO_BLOCK if the product info block
  *                                           could not be read.
- *         MARCHANNEL_MISMATCH_ERROR         if update-settings.ini's MAR 
+ *         MARCHANNEL_MISMATCH_ERROR         if update-settings.ini's MAR
  *                                           channel ID doesn't match the MAR
- *                                           file's MAR channel ID. 
+ *                                           file's MAR channel ID.
  *         VERSION_DOWNGRADE_ERROR           if the application version for
  *                                           this updater is newer than the
  *                                           one in the MAR.
  */
 int
-ArchiveReader::VerifyProductInformation(const char *MARChannelID, 
+ArchiveReader::VerifyProductInformation(const char *MARChannelID,
                                         const char *appVersion)
 {
   if (!mArchive) {
     return ARCHIVE_NOT_OPEN;
   }
 
   ProductInformationBlock productInfoBlock;
-  int rv = mar_read_product_info_block(mArchive, 
+  int rv = mar_read_product_info_block(mArchive,
                                        &productInfoBlock);
   if (rv != OK) {
     return COULD_NOT_READ_PRODUCT_INFO_BLOCK_ERROR;
   }
 
   // Only check the MAR channel name if specified, it should be passed in from
   // the update-settings.ini file.
   if (MARChannelID && strlen(MARChannelID)) {
     // Check for at least one match in the comma separated list of values.
     const char *delimiter = " ,\t";
-    // Make a copy of the string in case a read only memory buffer 
+    // Make a copy of the string in case a read only memory buffer
     // was specified.  strtok modifies the input buffer.
     char channelCopy[512] = { 0 };
     strncpy(channelCopy, MARChannelID, sizeof(channelCopy) - 1);
     char *channel = strtok(channelCopy, delimiter);
     rv = MAR_CHANNEL_MISMATCH_ERROR;
     while(channel) {
       if (!strcmp(channel, productInfoBlock.MARChannelID)) {
         rv = OK;
@@ -158,17 +163,17 @@ ArchiveReader::VerifyProductInformation(
         -1 if appVersion is older than productInfoBlock.productVersion
         1 if appVersion is newer than productInfoBlock.productVersion
         0 if appVersion is the same as productInfoBlock.productVersion
        This even works with strings like:
         - 12.0a1 being older than 12.0a2
         - 12.0a2 being older than 12.0b1
         - 12.0a1 being older than 12.0
         - 12.0 being older than 12.1a1 */
-    int versionCompareResult = 
+    int versionCompareResult =
       mozilla::CompareVersions(appVersion, productInfoBlock.productVersion);
     if (1 == versionCompareResult) {
       rv = VERSION_DOWNGRADE_ERROR;
     }
   }
 
   free((void *)productInfoBlock.MARChannelID);
   free((void *)productInfoBlock.productVersion);
@@ -198,24 +203,49 @@ ArchiveReader::Open(const NS_tchar *path
       // Try again with a smaller buffer.
       outbuf_size = 1024;
       outbuf = (char *)malloc(outbuf_size);
       if (!outbuf)
         return ARCHIVE_READER_MEM_ERROR;
     }
   }
 
+  // Find out if the archive file is compressed,
+  // and decompress it before handing it off to libmar.
+  FILE * archive = NS_tfopen(path, NS_T("rb"));
+  if (!archive) {
+    return READ_ERROR;
+  }
+  if (fread(inbuf, sizeof(XZ_MAGIC), 1, archive) != 1) {
+    fclose(archive);
+    return READ_ERROR;
+  }
+
+  NS_tchar newPath[MAXPATHLEN];
+  if (memcmp(inbuf, XZ_MAGIC, sizeof(XZ_MAGIC)) == 0) {
+    NS_tstrcpy(newPath, path);
+    NS_tstrncpy(newPath + NS_tstrlen(path), NS_T(".extracted"),
+                MAXPATHLEN - NS_tstrlen(path));
+
+    int decompressResult = DecompressXZArchive(archive, newPath);
+    if (decompressResult != OK) {
+      return decompressResult;
+    }
+    path = newPath;
+  }
+
+
 #ifdef XP_WIN
   mArchive = mar_wopen(path);
 #else
   mArchive = mar_open(path);
 #endif
-  if (!mArchive)
+  if (!mArchive) {
     return READ_ERROR;
-
+  }
   return OK;
 }
 
 void
 ArchiveReader::Close()
 {
   if (mArchive) {
     mar_close(mArchive);
@@ -229,16 +259,75 @@ ArchiveReader::Close()
 
   if (outbuf) {
     free(outbuf);
     outbuf = nullptr;
   }
 }
 
 int
+ArchiveReader::DecompressXZArchive(FILE *compressed, const NS_tchar *decompressedPath)
+{
+#ifdef XP_WIN
+  FILE* decompressed = _wfopen(decompressedPath, L"wb+");
+#else
+  FILE *decompressed = fopen(decompressedPath, "wb+");
+#endif
+  if (!decompressed) {
+    return WRITE_ERROR_EXTRACT;
+  }
+
+  // 64 MB is the *maximum* dictionary size; the decoder will allocate up to
+  // that amount, but only if it really needs that much.
+  xz_dec * dec = xz_dec_init(XZ_DYNALLOC, 64 * 1024 * 1024);
+  if (dec == nullptr) {
+    return ARCHIVE_READER_MEM_ERROR;
+  }
+
+  xz_buf strm = { 0 };
+  strm.in = (uint8_t*)inbuf;
+  strm.out = (uint8_t*)outbuf;
+  strm.out_size = outbuf_size;
+
+  fseek(compressed, 0, SEEK_SET);
+
+  int rv = OK;
+  xz_ret xz_rv = XZ_OK;
+  while (true) {
+    if (strm.in_pos >= strm.in_size) {
+      size_t inlen = fread(inbuf, 1, inbuf_size, compressed);
+      if (inlen == 0) {
+        rv = feof(compressed) && xz_rv == XZ_STREAM_END ? OK : READ_ERROR;
+        break;
+      }
+      strm.in_size = inlen;
+    }
+
+    strm.in_pos = 0;
+    strm.out_pos = 0;
+    xz_rv = xz_dec_run(dec, &strm);
+    if (xz_rv != XZ_OK && xz_rv != XZ_STREAM_END) {
+      rv = UNEXPECTED_XZ_ERROR;
+      break;
+    }
+
+    if (strm.out_pos != 0) {
+      if (fwrite(outbuf, strm.out_pos, 1, decompressed) != 1) {
+        rv = WRITE_ERROR_EXTRACT;
+        break;
+      }
+    }
+  }
+
+  xz_dec_end(dec);
+  fclose(decompressed);
+  return rv;
+}
+
+int
 ArchiveReader::ExtractFile(const char *name, const NS_tchar *dest)
 {
   const MarItem *item = mar_find_item(mArchive, name);
   if (!item)
     return READ_ERROR;
 
 #ifdef XP_WIN
   FILE* fp = _wfopen(dest, L"wb+");
@@ -266,16 +355,49 @@ ArchiveReader::ExtractFileToStream(const
     return READ_ERROR;
 
   return ExtractItemToStream(item, fp);
 }
 
 int
 ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp)
 {
+  if (mar_read(mArchive, item, 0, inbuf, sizeof(BZ2_MAGIC)) <
+      (int)sizeof(BZ2_MAGIC)) {
+    return READ_ERROR;
+  }
+
+  if (memcmp(inbuf, BZ2_MAGIC, sizeof(BZ2_MAGIC)) == 0) {
+    return ExtractBZ2ItemToStream(item, fp);
+  } else {
+    return ExtractRawItemToStrean(item, fp);
+  }
+}
+
+int
+ArchiveReader::ExtractRawItemToStrean(const MarItem *item, FILE *fp)
+{
+  for (int offset = 0; offset < (int)item->length;) {
+    int inlen = mar_read(mArchive, item, offset, inbuf, inbuf_size);
+    if (inlen <= 0) {
+      return READ_ERROR;
+    }
+    offset += inlen;
+
+    if (fwrite(inbuf, inlen, 1, fp) != 1) {
+      return WRITE_ERROR_EXTRACT;
+    }
+  }
+
+  return OK;
+}
+
+int
+ArchiveReader::ExtractBZ2ItemToStream(const MarItem *item, FILE *fp)
+{
   /* decompress the data chunk by chunk */
 
   bz_stream strm;
   int offset, inlen, outlen, ret = OK;
 
   memset(&strm, 0, sizeof(strm));
   if (BZ2_bzDecompressInit(&strm, 0, 0) != BZ_OK)
     return UNEXPECTED_BZIP_ERROR;
--- a/toolkit/mozapps/update/updater/archivereader.h
+++ b/toolkit/mozapps/update/updater/archivereader.h
@@ -28,14 +28,18 @@ public:
   int VerifyProductInformation(const char *MARChannelID, 
                                const char *appVersion);
   void Close();
 
   int ExtractFile(const char *item, const NS_tchar *destination);
   int ExtractFileToStream(const char *item, FILE *fp);
 
 private:
+  int DecompressXZArchive(FILE *compressed, const NS_tchar *decompressedPath);
+
   int ExtractItemToStream(const MarItem *item, FILE *fp);
+  int ExtractRawItemToStrean(const MarItem *item, FILE *fp);
+  int ExtractBZ2ItemToStream(const MarItem *item, FILE *fp);
 
   MarFile *mArchive;
 };
 
 #endif  // ArchiveReader_h__
--- a/toolkit/mozapps/update/updater/updater-common.build
+++ b/toolkit/mozapps/update/updater/updater-common.build
@@ -62,16 +62,21 @@ USE_LIBS += [
 
 if CONFIG['MOZ_SYSTEM_BZ2']:
     OS_LIBS += CONFIG['MOZ_BZ2_LIBS']
 else:
     USE_LIBS += [
         'bz2',
     ]
 
+USE_LIBS += [
+    'xz-embedded',
+]
+DEFINES['XZ_USE_CRC64'] = 1
+
 if 'gtk' in CONFIG['MOZ_WIDGET_TOOLKIT']:
     have_progressui = 1
     srcs += [
         'progressui_gtk.cpp',
     ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa':
     have_progressui = 1
--- a/toolkit/mozapps/update/updater/updater.cpp
+++ b/toolkit/mozapps/update/updater/updater.cpp
@@ -34,16 +34,17 @@
  *  method   = "remove" | "rmdir"
  */
 #include "bspatch.h"
 #include "progressui.h"
 #include "archivereader.h"
 #include "readstrings.h"
 #include "errors.h"
 #include "bzlib.h"
+#include "xz.h"
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdarg.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -2736,16 +2737,20 @@ int LaunchCallbackAndPostProcessApps(int
     } // if (!isElevated)
 #endif /* XP_MACOSX */
   }
   return 0;
 }
 
 int NS_main(int argc, NS_tchar **argv)
 {
+  // Have to call these once before we can extract XZ files.
+  xz_crc32_init();
+  xz_crc64_init();
+
   // The callback is the remaining arguments starting at callbackIndex.
   // The argument specified by callbackIndex is the callback executable and the
   // argument prior to callbackIndex is the working directory.
   const int callbackIndex = 6;
 
 #ifdef XP_MACOSX
   bool isElevated =
     strstr(argv[0], "/Library/PrivilegedHelperTools/org.mozilla.updater") != 0;