Bug 641212 Part 5 - Support opening and extracting XZ-compressed MAR archives. r?rstrong draft
authorMatt Howell <mhowell@mozilla.com>
Thu, 26 Jan 2017 14:58:07 -0800
changeset 607657 1cba5ce0b41469b70b0effd8743505157e26aa06
parent 607656 1c0cbaadc6768afd981953f84cf1b164d1376af9
child 607658 f40d4bc50c3546e508771e0214401242f54391e1
push id68072
push usermhowell@mozilla.com
push dateWed, 12 Jul 2017 17:32:12 +0000
reviewersrstrong
bugs641212
milestone56.0a1
Bug 641212 Part 5 - Support opening and extracting XZ-compressed MAR archives. r?rstrong MozReview-Commit-ID: H5cGeYJmYiK
modules/libmar/src/mar.h
modules/libmar/src/mar_decompress.c
modules/libmar/src/mar_extract.c
modules/libmar/src/mar_private.h
modules/libmar/src/mar_read.c
modules/libmar/src/moz.build
--- a/modules/libmar/src/mar.h
+++ b/modules/libmar/src/mar.h
@@ -42,16 +42,22 @@ typedef struct MarItem_ {
   uint32_t flags;         /* contains file mode bits */
   char name[1];           /* file path */
 } MarItem;
 
 #define TABLESIZE 256
 
 struct MarFile_ {
   FILE *fp;
+  bool decompressed;
+#ifdef XP_WIN
+  wchar_t* name;
+#else
+  char* name;
+#endif
   MarItem *item_table[TABLESIZE];
 };
 
 typedef struct MarFile_ MarFile;
 
 /**
  * Signature of callback function passed to mar_enum_items.
  * @param mar       The MAR file being visited.
@@ -186,13 +192,34 @@ int mar_verify_signatures(MarFile *mar,
  *
  * @param infoBlock Out parameter for where to store the result to
  * @return 0 on success, -1 on failure
 */
 int
 mar_read_product_info_block(MarFile *mar, 
                             struct ProductInformationBlock *infoBlock);
 
+/**
+ * Finds the start and length of the content area of the MAR file.
+ *
+ * @param      mar     The already opened MAR file
+ * @param[out] offset  File offset in bytes of the start of the content,
+ *                     or -1 if there is no content
+ * @param[out] length  Total length of all content files
+*/
+void
+mar_get_content_extent(MarFile* mar, int* offset, int* length);
+
+/**
+* Decompress the content area of a MAR file in place.
+*
+* @param      mar     The already opened MAR file
+*
+* @return 0 if successful, 1 if the MAR is not compressed, -1 on error
+*/
+int
+mar_decompress(MarFile* mar);
+
 #ifdef __cplusplus
 }
 #endif
 
 #endif  /* MAR_H__ */
new file mode 100644
--- /dev/null
+++ b/modules/libmar/src/mar_decompress.c
@@ -0,0 +1,180 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// for ntohl
+#ifdef XP_WIN
+#include <winsock2.h>
+#else
+#include <netinet/in.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include "updatedefines.h"
+#include "mar_private.h"
+#include "lzma.h"
+
+#define FILE_BUFFER_SIZE 262144
+static uint8_t inBuf[FILE_BUFFER_SIZE];
+static uint8_t outBuf[FILE_BUFFER_SIZE];
+
+static int
+mar_decompress_xz_archive(FILE* from, FILE* to, int compressedSize)
+{
+  lzma_stream strm = LZMA_STREAM_INIT;
+  if (lzma_stream_decoder(&strm, UINT64_MAX, 0) != LZMA_OK) {
+    return -1;
+  }
+
+  strm.next_in = NULL;
+  strm.avail_in = 0;
+  strm.next_out = outBuf;
+  strm.avail_out = FILE_BUFFER_SIZE;
+
+  int rv = 0;
+  lzma_ret lzma_rv = LZMA_OK;
+  while (lzma_rv == LZMA_OK) {
+    lzma_action action = LZMA_RUN;
+    if (strm.avail_in == 0) {
+      strm.next_in = inBuf;
+      strm.avail_in = fread(inBuf, 1, FILE_BUFFER_SIZE, from);
+      if (ferror(from)) {
+        rv = -1;
+        break;
+      }
+      if (strm.total_in + strm.avail_in >= (uint64_t)compressedSize) {
+        action = LZMA_FINISH;
+      }
+    }
+
+    lzma_rv = lzma_code(&strm, action);
+    if (strm.avail_out == 0 || lzma_rv == LZMA_STREAM_END) {
+      size_t writeBytes = FILE_BUFFER_SIZE - strm.avail_out;
+      if (fwrite(outBuf, 1, writeBytes, to) != writeBytes) {
+        rv = -1;
+        break;
+      }
+      strm.next_out = outBuf;
+      strm.avail_out = FILE_BUFFER_SIZE;
+    }
+  }
+
+  lzma_end(&strm);
+  if (lzma_rv != LZMA_STREAM_END) {
+    rv = -1;
+  }
+  return rv;
+}
+
+static int
+mar_is_compressed(FILE* file)
+{
+  static const char XZ_MAGIC[] = { 0xFD, '7', 'z', 'X', 'Z', 0x00 };
+  char buffer[sizeof(XZ_MAGIC)];
+  if (fread(buffer, sizeof(XZ_MAGIC), 1, file) != 1) {
+    return 0;
+  }
+  fseek(file, -(int)sizeof(XZ_MAGIC), SEEK_CUR);
+  return (memcmp(buffer, XZ_MAGIC, sizeof(XZ_MAGIC)) == 0) ? 1 : 0;
+}
+
+int
+mar_decompress(MarFile* file)
+{
+  int contentOffset = 0, contentLength = 0;
+  mar_get_content_extent(file, &contentOffset, &contentLength);
+
+  fseek(file->fp, contentOffset, SEEK_SET);
+
+  if (!mar_is_compressed(file->fp)) {
+    return 1;
+  }
+
+  // Create a temporary archive file.
+  NS_tchar extractedPath[MAXPATHLEN] = NS_T("");
+  int sprintfLen = NS_tsnprintf(extractedPath, MAXPATHLEN, NS_T("%s.extracted"), file->name);
+  // Make sure the path wasn't truncated before opening the file.
+  if (sprintfLen != (int)NS_tstrlen(extractedPath)) {
+    return -1;
+  }
+  FILE * extractedFile = NS_tfopen(extractedPath, NS_T("wb+"));
+  if (!extractedFile) {
+    return -1;
+  }
+
+  // Copy from the archive file into the temp file, up to the content area.
+  int rv = 0;
+  int bytesRead = 0;
+  fseek(file->fp, 0, SEEK_SET);
+  while (bytesRead < contentOffset) {
+    int bytesToRead = FILE_BUFFER_SIZE;
+    if ((contentOffset - bytesRead) < FILE_BUFFER_SIZE) {
+      bytesToRead = contentOffset - bytesRead;
+    }
+    if (fread(inBuf, 1, bytesToRead, file->fp) == 0) {
+      rv = -1;
+      break;
+    }
+    bytesRead += bytesToRead;
+    if (fwrite(outBuf, bytesToRead, 1, extractedFile) != 1) {
+      rv = -1;
+      break;
+    }
+  }
+  if (rv) {
+    fclose(extractedFile);
+    NS_tremove(extractedPath);
+    return rv;
+  }
+
+  // Extract the compressed content into the temp file.
+  if (mar_decompress_xz_archive(file->fp, extractedFile, contentLength) != 0) {
+    fclose(extractedFile);
+    NS_tremove(extractedPath);
+    return -1;
+  }
+
+  // Reset the file stream to the end of the content, because the extraction
+  // might have read past the end of the actual XZ stream.
+  fseek(file->fp, contentOffset + contentLength, SEEK_SET);
+
+  // Copy over the index from the MAR, which is all that's left past the content.
+  while (1) {
+    size_t bytesRead = fread(inBuf, 1, FILE_BUFFER_SIZE, file->fp);
+    if (bytesRead == 0) {
+      rv = feof(file->fp) ? 0 : -1;
+      break;
+    }
+    if (fwrite(outBuf, bytesRead, 1, extractedFile) != 1) {
+      rv = -1;
+      break;
+    }
+  }
+  if (rv) {
+    fclose(extractedFile);
+    NS_tremove(extractedPath);
+    return rv;
+  }
+
+  fclose(file->fp);
+  file->fp = extractedFile;
+  file->decompressed = true;
+  return 0;
+}
+
+void
+mar_decompress_cleanup(MarFile* file)
+{
+  NS_tchar extractedPath[MAXPATHLEN];
+  int sprintfLen =
+    NS_tsnprintf(extractedPath, MAXPATHLEN, NS_T("%s.extracted"), file->name);
+  // Make sure the path wasn't truncated before removing the file.
+  if (sprintfLen == (int)NS_tstrlen(extractedPath)) {
+    NS_tremove(extractedPath);
+  }
+}
--- a/modules/libmar/src/mar_extract.c
+++ b/modules/libmar/src/mar_extract.c
@@ -68,16 +68,20 @@ static int mar_test_callback(MarFile *ma
   return len == 0 ? 0 : -1;
 }
 
 int mar_extract(const char *path) {
   MarFile *mar;
   int rv;
 
   mar = mar_open(path);
-  if (!mar)
+  if (!mar) {
     return -1;
+  }
 
-  rv = mar_enum_items(mar, mar_test_callback, NULL);
+  rv = mar_decompress(mar);
+  if (!rv) {
+    rv = mar_enum_items(mar, mar_test_callback, NULL);
+  }
 
   mar_close(mar);
   return rv;
 }
--- a/modules/libmar/src/mar_private.h
+++ b/modules/libmar/src/mar_private.h
@@ -2,16 +2,17 @@
 /* vim:set ts=2 sw=2 sts=2 et cindent: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef MAR_PRIVATE_H__
 #define MAR_PRIVATE_H__
 
+#include "mar.h"
 #include "limits.h"
 #include "mozilla/Assertions.h"
 #include <stdint.h>
 
 #define BLOCKSIZE 4096
 #define ROUND_UP(n, incr) (((n) / (incr) + 1) * (incr))
 
 #define MAR_ID "MAR1"
@@ -71,9 +72,16 @@ MOZ_STATIC_ASSERT(sizeof(BLOCKSIZE) < \
   (((((uint64_t) x) >> 16) & 0xFF) << 40) | \
   (((((uint64_t) x) >> 24) & 0xFF) << 32) | \
   (((((uint64_t) x) >> 32) & 0xFF) << 24) | \
   (((((uint64_t) x) >> 40) & 0xFF) << 16) | \
   (((((uint64_t) x) >> 48) & 0xFF) << 8) | \
   (((uint64_t) x) >> 56)
 #define NETWORK_TO_HOST64 HOST_TO_NETWORK64
 
+/**
+ * Clean up after mar_decompress().
+ * Should be called after the MarFile fp has been closed.
+ */
+void
+mar_decompress_cleanup(MarFile* file);
+
 #endif  /* MAR_PRIVATE_H__ */
--- a/modules/libmar/src/mar_read.c
+++ b/modules/libmar/src/mar_read.c
@@ -155,16 +155,18 @@ static MarFile *mar_fpopen(FILE *fp)
 
   mar = (MarFile *) malloc(sizeof(*mar));
   if (!mar) {
     fclose(fp);
     return NULL;
   }
 
   mar->fp = fp;
+  mar->decompressed = false;
+  mar->name = NULL;
   memset(mar->item_table, 0, sizeof(mar->item_table));
   if (mar_read_index(mar)) {
     mar_close(mar);
     return NULL;
   }
 
   return mar;
 }
@@ -174,31 +176,51 @@ MarFile *mar_open(const char *path) {
 
   fp = fopen(path, "rb");
   if (!fp) {
     fprintf(stderr, "ERROR: could not open file in mar_open()\n");
     perror(path);
     return NULL;
   }
 
-  return mar_fpopen(fp);
+  MarFile *rv = mar_fpopen(fp);
+  if (rv) {
+#ifdef XP_WIN
+    int pathLen = MultiByteToWideChar(CP_UTF8, 0, path, -1, NULL, 0);
+    rv->name = malloc(pathLen * sizeof(wchar_t));
+    MultiByteToWideChar(CP_UTF8, 0, path, -1, rv->name, pathLen);
+#else
+    rv->name = malloc(strlen(path) + 1);
+    strcpy(rv->name, path);
+#endif
+    if (!rv->name) {
+      mar_close(rv);
+      rv = NULL;
+    }
+  }
+  return rv;
 }
 
 #ifdef XP_WIN
 MarFile *mar_wopen(const wchar_t *path) {
   FILE *fp;
 
   _wfopen_s(&fp, path, L"rb");
   if (!fp) {
     fprintf(stderr, "ERROR: could not open file in mar_wopen()\n");
     _wperror(path);
     return NULL;
   }
 
-  return mar_fpopen(fp);
+  MarFile *rv = mar_fpopen(fp);
+  if (rv) {
+    rv->name = malloc((wcslen(path) + 1) * sizeof(wchar_t));
+    wcscpy(rv->name, path);
+  }
+  return rv;
 }
 #endif
 
 void mar_close(MarFile *mar) {
   MarItem *item;
   int i;
 
   fclose(mar->fp);
@@ -207,16 +229,21 @@ void mar_close(MarFile *mar) {
     item = mar->item_table[i];
     while (item) {
       MarItem *temp = item;
       item = item->next;
       free(temp);
     }
   }
 
+  if (mar->decompressed) {
+    mar_decompress_cleanup(mar);
+  }
+
+  free(mar->name);
   free(mar);
 }
 
 /**
  * Determines the MAR file information.
  *
  * @param fp                     An opened MAR file in read mode.
  * @param hasSignatureBlock      Optional out parameter specifying if the MAR
@@ -570,8 +597,36 @@ int get_mar_file_info(const char *path,
 
   rv = get_mar_file_info_fp(fp, hasSignatureBlock, 
                             numSignatures, hasAdditionalBlocks,
                             offsetAdditionalBlocks, numAdditionalBlocks);
 
   fclose(fp);
   return rv;
 }
+
+
+void
+mar_get_content_extent(MarFile* mar,
+                       int* offset,
+                       int* length)
+{
+  // The offset to the end of the content area is just the offset to the
+  // index listed in the file header, because the index begins immediately
+  // following the content.
+  fseek(mar->fp, MAR_ID_SIZE, SEEK_SET);
+  int endOfContent = 0;
+  if (fread(&endOfContent, sizeof(endOfContent), 1, mar->fp) != 1) {
+    return;
+  }
+  endOfContent = ntohl(endOfContent);
+
+  // The start of the content is always the start of the first file listed in
+  // the index, because the index is always written in order.
+  fseek(mar->fp, endOfContent + 4, SEEK_SET);
+  int startOfContent = 0;
+  if (fread(&startOfContent, sizeof(startOfContent), 1, mar->fp) != 1) {
+    return;
+  }
+
+  *offset = ntohl(startOfContent);
+  *length = endOfContent - startOfContent;
+}
--- a/modules/libmar/src/moz.build
+++ b/modules/libmar/src/moz.build
@@ -6,28 +6,34 @@
 
 EXPORTS += [
     'mar.h',
     'mar_cmdline.h',
 ]
 
 HOST_SOURCES += [
     'mar_create.c',
+    'mar_decompress.c',
     'mar_extract.c',
     'mar_read.c',
 ]
 HostLibrary('hostmar')
 
 Library('mar')
 
 UNIFIED_SOURCES += [
     'mar_create.c',
+    'mar_decompress.c',
     'mar_extract.c',
     'mar_read.c',
 ]
 
+LOCAL_INCLUDES += [
+    '/toolkit/mozapps/update/common',
+]
+
 FORCE_STATIC_LIB = True
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     USE_STATIC_LIBS = True
 
 DEFINES['LZMA_API_STATIC'] = 1
 HOST_DEFINES['LZMA_API_STATIC'] = 1