Bug 934967 - Part 1: Compress session store files using lz4. r?mikedeboer draft
authorBeekill95 <nnn_bikiu0707@yahoo.com>
Thu, 08 Jun 2017 15:14:18 +0700
changeset 597191 a375fc4e2d7262848f65c35b601cc6748d088ccc
parent 596219 95543bdc59bd038a3d5d084b85a4fec493c349ee
child 597192 69d62402f719ceab08b903adcbc6f823fcd9b283
push id64869
push userbmo:nnn_bikiu0707@yahoo.com
push dateTue, 20 Jun 2017 08:50:00 +0000
reviewersmikedeboer
bugs934967
milestone56.0a1
Bug 934967 - Part 1: Compress session store files using lz4. r?mikedeboer MozReview-Commit-ID: 6wKLIAglefr
browser/components/sessionstore/SessionFile.jsm
browser/components/sessionstore/SessionWorker.js
--- a/browser/components/sessionstore/SessionFile.jsm
+++ b/browser/components/sessionstore/SessionFile.jsm
@@ -93,46 +93,46 @@ Object.freeze(SessionFile);
 
 var Path = OS.Path;
 var profileDir = OS.Constants.Path.profileDir;
 
 var SessionFileInternal = {
   Paths: Object.freeze({
     // The path to the latest version of sessionstore written during a clean
     // shutdown. After startup, it is renamed `cleanBackup`.
-    clean: Path.join(profileDir, "sessionstore.js"),
+    clean: Path.join(profileDir, "sessionstore.jsonlz4"),
 
     // The path at which we store the previous version of `clean`. Updated
     // whenever we successfully load from `clean`.
-    cleanBackup: Path.join(profileDir, "sessionstore-backups", "previous.js"),
+    cleanBackup: Path.join(profileDir, "sessionstore-backups", "previous.jsonlz4"),
 
     // The directory containing all sessionstore backups.
     backups: Path.join(profileDir, "sessionstore-backups"),
 
     // The path to the latest version of the sessionstore written
     // during runtime. Generally, this file contains more
     // privacy-sensitive information than |clean|, and this file is
     // therefore removed during clean shutdown. This file is designed to protect
     // against crashes / sudden shutdown.
-    recovery: Path.join(profileDir, "sessionstore-backups", "recovery.js"),
+    recovery: Path.join(profileDir, "sessionstore-backups", "recovery.jsonlz4"),
 
     // The path to the previous version of the sessionstore written
     // during runtime (e.g. 15 seconds before recovery). In case of a
     // clean shutdown, this file is removed.  Generally, this file
     // contains more privacy-sensitive information than |clean|, and
     // this file is therefore removed during clean shutdown.  This
     // file is designed to protect against crashes that are nasty
     // enough to corrupt |recovery|.
-    recoveryBackup: Path.join(profileDir, "sessionstore-backups", "recovery.bak"),
+    recoveryBackup: Path.join(profileDir, "sessionstore-backups", "recovery.baklz4"),
 
     // The path to a backup created during an upgrade of Firefox.
     // Having this backup protects the user essentially from bugs in
     // Firefox or add-ons, especially for users of Nightly. This file
     // does not contain any information more sensitive than |clean|.
-    upgradeBackupPrefix: Path.join(profileDir, "sessionstore-backups", "upgrade.js-"),
+    upgradeBackupPrefix: Path.join(profileDir, "sessionstore-backups", "upgrade.jsonlz4-"),
 
     // The path to the backup of the version of the session store used
     // during the latest upgrade of Firefox. During load/recovery,
     // this file should be used if both |path|, |backupPath| and
     // |latestStartPath| are absent/incorrect.  May be "" if no
     // upgrade backup has ever been performed. This file does not
     // contain any information more sensitive than |clean|.
     get upgradeBackup() {
@@ -202,42 +202,50 @@ var SessionFileInternal = {
   get latestUpgradeBackupID() {
     try {
       return Services.prefs.getCharPref(PREF_UPGRADE_BACKUP);
     } catch (ex) {
       return undefined;
     }
   },
 
-  // Find the correct session file, read it and setup the worker.
-  async read() {
-    this._initializationStarted = true;
-
+  async _readInternal(useOldExtension) {
     let result;
     let noFilesFound = true;
+
     // Attempt to load by order of priority from the various backups
     for (let key of this.Paths.loadOrder) {
       let corrupted = false;
       let exists = true;
       try {
-        let path = this.Paths[key];
+        let path;
         let startMs = Date.now();
 
-        let source = await OS.File.read(path, { encoding: "utf-8" });
+        let options = {encoding: "utf-8"};
+        if (useOldExtension) {
+          path = this.Paths[key]
+                     .replace("jsonlz4", "js")
+                     .replace("baklz4", "bak");
+        } else {
+          path = this.Paths[key];
+          options.compression = "lz4";
+        }
+        let source = await OS.File.read(path, options);
         let parsed = JSON.parse(source);
 
         if (!SessionStore.isFormatVersionCompatible(parsed.version || ["sessionrestore", 0] /* fallback for old versions*/)) {
           // Skip sessionstore files that we don't understand.
           Cu.reportError("Cannot extract data from Session Restore file " + path + ". Wrong format/version: " + JSON.stringify(parsed.version) + ".");
           continue;
         }
         result = {
           origin: key,
           source,
-          parsed
+          parsed,
+          useOldExtension
         };
         Telemetry.getHistogramById("FX_SESSION_RESTORE_CORRUPT_FILE").
           add(false);
         Telemetry.getHistogramById("FX_SESSION_RESTORE_READ_FILE_MS").
           add(Date.now() - startMs);
         break;
       } catch (ex) {
           if (ex instanceof OS.File.Error && ex.becauseNoSuchFile) {
@@ -255,36 +263,52 @@ var SessionFileInternal = {
       } finally {
         if (exists) {
           noFilesFound = false;
           Telemetry.getHistogramById("FX_SESSION_RESTORE_CORRUPT_FILE").
             add(corrupted);
         }
       }
     }
+    return {result, noFilesFound};
+  },
+
+  // Find the correct session file, read it and setup the worker.
+  async read() {
+    this._initializationStarted = true;
+
+    // Load session files with lz4 compression.
+    let {result, noFilesFound} = await this._readInternal(false);
+    if (!result) {
+      // No result? Probably because of migration, let's
+      // load uncompressed session files.
+      let r = await this._readInternal(true);
+      result = r.result;
+    }
 
     // All files are corrupted if files found but none could deliver a result.
     let allCorrupt = !noFilesFound && !result;
     Telemetry.getHistogramById("FX_SESSION_RESTORE_ALL_FILES_CORRUPT").
       add(allCorrupt);
 
     if (!result) {
       // If everything fails, start with an empty session.
       result = {
         origin: "empty",
         source: "",
-        parsed: null
+        parsed: null,
+        useOldExtension: false
       };
     }
 
     result.noFilesFound = noFilesFound;
 
     // Initialize the worker (in the background) to let it handle backups and also
     // as a workaround for bug 964531.
-    let promiseInitialized = SessionWorker.post("init", [result.origin, this.Paths, {
+    let promiseInitialized = SessionWorker.post("init", [result.origin, result.useOldExtension, this.Paths, {
       maxUpgradeBackups: Preferences.get(PREF_MAX_UPGRADE_BACKUPS, 3),
       maxSerializeBack: Preferences.get(PREF_MAX_SERIALIZE_BACK, 10),
       maxSerializeForward: Preferences.get(PREF_MAX_SERIALIZE_FWD, -1)
     }]);
 
     promiseInitialized.catch(err => {
       // Ensure that we report errors but that they do not stop us.
       Promise.reject(err);
--- a/browser/components/sessionstore/SessionWorker.js
+++ b/browser/components/sessionstore/SessionWorker.js
@@ -75,40 +75,47 @@ var Agent = {
    *   if we have started without any sessionstore;
    * - one of "clean", "recovery", "recoveryBackup", "cleanBackup",
    *   "upgradeBackup", before the first write has been completed, if
    *   we have started by loading the corresponding file.
    */
   state: null,
 
   /**
+   * A flag that indicates we loaded a session file with the deprecated .js extension.
+   */
+  useOldExtension: false,
+
+  /**
    * Number of old upgrade backups that are being kept
    */
   maxUpgradeBackups: null,
 
   /**
    * Initialize (or reinitialize) the worker
    *
    * @param {string} origin Which of sessionstore.js or its backups
    *   was used. One of the `STATE_*` constants defined above.
+   * @param {boolean} a flag indicate whether we loaded a session file with ext .js
    * @param {object} paths The paths at which to find the various files.
    * @param {object} prefs The preferences the worker needs to known.
    */
-  init(origin, paths, prefs = {}) {
+  init(origin, useOldExtension, paths, prefs = {}) {
     if (!(origin in paths || origin == STATE_EMPTY)) {
       throw new TypeError("Invalid origin: " + origin);
     }
 
     // Check that all required preference values were passed.
     for (let pref of ["maxUpgradeBackups", "maxSerializeBack", "maxSerializeForward"]) {
       if (!prefs.hasOwnProperty(pref)) {
         throw new TypeError(`Missing preference value for ${pref}`);
       }
     }
 
+    this.useOldExtension = useOldExtension;
     this.state = origin;
     this.Paths = paths;
     this.maxUpgradeBackups = prefs.maxUpgradeBackups;
     this.maxSerializeBack = prefs.maxSerializeBack;
     this.maxSerializeForward = prefs.maxSerializeForward;
     this.upgradeBackupNeeded = paths.nextUpgradeBackup != paths.upgradeBackup;
     return {result: true};
   },
@@ -160,52 +167,68 @@ var Agent = {
         // we have either already read from or already written to this
         // directory, so we are satisfied that it exists.
         File.makeDir(this.Paths.backups);
       }
 
       if (this.state == STATE_CLEAN) {
         // Move $Path.clean out of the way, to avoid any ambiguity as
         // to which file is more recent.
-        File.move(this.Paths.clean, this.Paths.cleanBackup);
+        if (!this.useOldExtension) {
+          File.move(this.Paths.clean, this.Paths.cleanBackup);
+        } else {
+          // Since we are migrating from .js to .jsonlz4,
+          // we need to compress the deprecated $Path.clean
+          // and write it to $Path.cleanBackup.
+          let oldCleanPath = this.Paths.clean.replace("jsonlz4", "js");
+          let d = File.read(oldCleanPath);
+          File.writeAtomic(this.Paths.cleanBackup, d, {compression: "lz4"});
+        }
       }
 
       let startWriteMs = Date.now();
+      let fileStat;
 
       if (options.isFinalWrite) {
         // We are shutting down. At this stage, we know that
         // $Paths.clean is either absent or corrupted. If it was
         // originally present and valid, it has been moved to
         // $Paths.cleanBackup a long time ago. We can therefore write
         // with the guarantees that we erase no important data.
         File.writeAtomic(this.Paths.clean, data, {
-          tmpPath: this.Paths.clean + ".tmp"
+          tmpPath: this.Paths.clean + ".tmp",
+          compression: "lz4"
         });
+        fileStat = File.stat(this.Paths.clean);
       } else if (this.state == STATE_RECOVERY) {
         // At this stage, either $Paths.recovery was written >= 15
         // seconds ago during this session or we have just started
         // from $Paths.recovery left from the previous session. Either
         // way, $Paths.recovery is good. We can move $Path.backup to
         // $Path.recoveryBackup without erasing a good file with a bad
         // file.
         File.writeAtomic(this.Paths.recovery, data, {
           tmpPath: this.Paths.recovery + ".tmp",
-          backupTo: this.Paths.recoveryBackup
+          backupTo: this.Paths.recoveryBackup,
+          compression: "lz4"
         });
+        fileStat = File.stat(this.Paths.recovery);
       } else {
         // In other cases, either $Path.recovery is not necessary, or
         // it doesn't exist or it has been corrupted. Regardless,
         // don't backup $Path.recovery.
         File.writeAtomic(this.Paths.recovery, data, {
-          tmpPath: this.Paths.recovery + ".tmp"
+          tmpPath: this.Paths.recovery + ".tmp",
+          compression: "lz4"
         });
+        fileStat = File.stat(this.Paths.recovery);
       }
 
       telemetry.FX_SESSION_RESTORE_WRITE_FILE_MS = Date.now() - startWriteMs;
-      telemetry.FX_SESSION_RESTORE_FILE_SIZE_BYTES = data.byteLength;
+      telemetry.FX_SESSION_RESTORE_FILE_SIZE_BYTES = fileStat.size;
 
     } catch (ex) {
       // Don't throw immediately
       exn = exn || ex;
     }
 
     // If necessary, perform an upgrade backup
     let upgradeBackupComplete = false;
@@ -288,16 +311,18 @@ var Agent = {
   wipe() {
 
     // Don't stop immediately in case of error.
     let exn = null;
 
     // Erase main session state file
     try {
       File.remove(this.Paths.clean);
+      // Remove old extension ones.
+      File.remove(this.Paths.clean.replace("jsonlz4", "js"), {ignoreAbsent: true});
     } catch (ex) {
       // Don't stop immediately.
       exn = exn || ex;
     }
 
     // Wipe the Session Restore directory
     try {
       this._wipeFromDir(this.Paths.backups, null);