Bug 1364768: Part 4 - Switch to a compressed, binary flat file for startup cache. r?aswan draft
authorKris Maglione <maglione.k@gmail.com>
Sun, 14 May 2017 16:06:30 -0700
changeset 577573 c0855428c4a1106dc01e0721364700e5fd97e2f0
parent 577572 a965c65dccbc7e570d74fb5007242a3197ec52b4
child 577574 96ee56aebb0892233e65429802b5709d1d8409b4
push id58718
push usermaglione.k@gmail.com
push dateSun, 14 May 2017 23:25:47 +0000
reviewersaswan
bugs1364768
milestone55.0a1
Bug 1364768: Part 4 - Switch to a compressed, binary flat file for startup cache. r?aswan IndexedDB helped where we needed to decrease main thread CPU, but it also took so long to inialize during startup (over 500ms on a fast machine) that it delayed extension startup more than was acceptable. Using a structured clone flat file solves the same issues that IndexedDB did, but with much less startup overhead. MozReview-Commit-ID: 1Of7uxKCfkg
toolkit/components/extensions/ExtensionManagement.jsm
toolkit/components/extensions/ExtensionUtils.jsm
--- a/toolkit/components/extensions/ExtensionManagement.jsm
+++ b/toolkit/components/extensions/ExtensionManagement.jsm
@@ -174,27 +174,17 @@ var Service = {
   // attached to the URI.
   extensionURIToAddonID(uri) {
     let uuid = uri.host;
     let extension = this.uuidMap.get(uuid);
     return extension ? extension.id : undefined;
   },
 };
 
-let cacheInvalidated = 0;
-function onCacheInvalidate() {
-  cacheInvalidated++;
-}
-Services.obs.addObserver(onCacheInvalidate, "startupcache-invalidate");
-
 ExtensionManagement = {
-  get cacheInvalidated() {
-    return cacheInvalidated;
-  },
-
   get isExtensionProcess() {
     if (this.useRemoteWebExtensions) {
       return appinfo.remoteType === E10SUtils.EXTENSION_REMOTE_TYPE;
     }
     return isParentProcess;
   },
 
   startupExtension: Service.startupExtension.bind(Service),
--- a/toolkit/components/extensions/ExtensionUtils.jsm
+++ b/toolkit/components/extensions/ExtensionUtils.jsm
@@ -17,27 +17,34 @@ Cu.import("resource://gre/modules/Servic
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
                                   "resource://gre/modules/AddonManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "AppConstants",
                                   "resource://gre/modules/AppConstants.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ConsoleAPI",
                                   "resource://gre/modules/Console.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "DeferredSave",
+                                  "resource://gre/modules/DeferredSave.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ExtensionManagement",
                                   "resource://gre/modules/ExtensionManagement.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "IndexedDB",
-                                  "resource://gre/modules/IndexedDB.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
+                                  "resource://gre/modules/FileUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "MessageChannel",
                                   "resource://gre/modules/MessageChannel.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
+                                  "resource://gre/modules/NetUtil.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Preferences",
                                   "resource://gre/modules/Preferences.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Schemas",
                                   "resource://gre/modules/Schemas.jsm");
 
+XPCOMUtils.defineLazyServiceGetter(this, "aomStartup",
+                                   "@mozilla.org/addons/addon-manager-startup;1",
+                                   "amIAddonManagerStartup");
 XPCOMUtils.defineLazyServiceGetter(this, "styleSheetService",
                                    "@mozilla.org/content/style-sheet-service;1",
                                    "nsIStyleSheetService");
 
 /* globals IDBKeyRange */
 
 function getConsole() {
   return new ConsoleAPI({
@@ -50,139 +57,161 @@ XPCOMUtils.defineLazyGetter(this, "conso
 
 let nextId = 0;
 XPCOMUtils.defineLazyGetter(this, "uniqueProcessID", () => Services.appinfo.uniqueProcessID);
 
 function getUniqueId() {
   return `${nextId++}-${uniqueProcessID}`;
 }
 
+function promiseFileContents(file) {
+  return new Promise((resolve, reject) => {
+    NetUtil.asyncFetch({uri: file, loadUsingSystemPrincipal: true}, (inputStream, status) => {
+      try {
+        if (!Components.isSuccessCode(status)) {
+          // Convert status code to a string
+          let e = Components.Exception("", status);
+          reject(new Error(`Error while loading '${file.path}' (${e.name})`));
+        } else {
+          resolve(NetUtil.readInputStream(inputStream));
+        }
+      } catch (e) {
+        reject(e);
+      }
+    });
+  });
+}
+
 let StartupCache = {
   DB_NAME: "ExtensionStartupCache",
 
-  SCHEMA_VERSION: 2,
-
   STORE_NAMES: Object.freeze(["locales", "manifests", "schemas"]),
 
-  dbPromise: null,
+  get file() {
+    return FileUtils.getFile("ProfLD", ["startupCache", "webext.sc.lz4"]);
+  },
 
-  cacheInvalidated: 0,
+  get saver() {
+    if (!this._saver) {
+      this._saver = new DeferredSave(this.file.path,
+                                     () => this.getBlob(),
+                                     {delay: 5000});
+    }
+    return this._saver;
+  },
+
+  async save() {
+    return this.saver.saveChanges();
+  },
+
+  getBlob() {
+    return new Uint8Array(aomStartup.encodeBlob(this._data));
+  },
 
-  initDB(db) {
-    for (let name of StartupCache.STORE_NAMES) {
-      try {
-        db.deleteObjectStore(name);
-      } catch (e) {
-        // Don't worry if the store doesn't already exist.
-      }
-      db.createObjectStore(name, {keyPath: "key"});
+  _data: null,
+  async _readData() {
+    let result = new Map();
+    try {
+      let data = await promiseFileContents(this.file);
+
+      result = aomStartup.decodeBlob(data);
+    } catch (e) {
+      Cu.reportError(e);
     }
+
+    this._data = result;
+    return result;
+  },
+
+  get dataPromise() {
+    if (!this._dataPromise) {
+      this._dataPromise = this._readData();
+    }
+    return this._dataPromise;
   },
 
   clearAddonData(id) {
-    let range = IDBKeyRange.bound([id], [id, "\uFFFF"]);
-
     return Promise.all([
-      this.locales.delete(range),
-      this.manifests.delete(range),
+      this.locales.delete(id),
+      this.manifests.delete(id),
     ]).catch(e => {
       // Ignore the error. It happens when we try to flush the add-on
       // data after the AddonManager has flushed the entire startup cache.
     });
   },
 
-  async reallyOpen(invalidate = false) {
-    if (this.dbPromise) {
-      let db = await this.dbPromise;
-      db.close();
-    }
-
-    if (invalidate) {
-      this.cacheInvalidated = ExtensionManagement.cacheInvalidated;
-
-      if (Services.appinfo.processType === Services.appinfo.PROCESS_TYPE_DEFAULT) {
-        IndexedDB.deleteDatabase(this.DB_NAME, {storage: "persistent"});
-      }
-    }
-
-    return IndexedDB.open(this.DB_NAME,
-                          {storage: "persistent", version: this.SCHEMA_VERSION},
-                          db => this.initDB(db));
-  },
-
-  async open() {
-    if (ExtensionManagement.cacheInvalidated > this.cacheInvalidated) {
-      this.dbPromise = this.reallyOpen(true);
-    } else if (!this.dbPromise) {
-      this.dbPromise = this.reallyOpen();
-    }
-
-    return this.dbPromise;
-  },
-
   observe(subject, topic, data) {
     if (topic === "startupcache-invalidate") {
-      this.dbPromise = this.reallyOpen(true).catch(e => {});
+      this._data = new Map();
+      this._dataPromise = Promise.resolve(this._data);
     }
   },
 };
 
-Services.obs.addObserver(StartupCache, "startupcache-invalidate");
+if (Services.appinfo.processType == Services.appinfo.PROCESS_TYPE_DEFAULT) {
+  void StartupCache.dataPromise;
+
+  Services.obs.addObserver(StartupCache, "startupcache-invalidate");
+}
 
 class CacheStore {
   constructor(storeName) {
     this.storeName = storeName;
   }
 
-  async get(key, createFunc) {
-    let db;
-    let result;
-    try {
-      db = await StartupCache.open();
+  async getStore(path = null) {
+    let data = await StartupCache.dataPromise;
+
+    let store = data.get(this.storeName);
+    if (!store) {
+      store = new Map();
+      data.set(this.storeName, store);
+    }
 
-      result = await db.objectStore(this.storeName)
-                      .get(key);
-    } catch (e) {
-      Cu.reportError(e);
+    let key = path;
+    if (Array.isArray(path)) {
+      for (let elem of path.slice(0, -1)) {
+        let next = store.get(elem);
+        if (!next) {
+          next = new Map();
+          store.set(elem, next);
+        }
+        store = next;
+      }
+      key = path[path.length - 1];
+    }
 
-      return createFunc(key);
-    }
+    return [store, key];
+  }
+
+  async get(path, createFunc) {
+    let [store, key] = await this.getStore(path);
+
+    let result = store.get(key);
 
     if (result === undefined) {
-      let value = await createFunc(key);
-      result = {key, value};
-
-      db.objectStore(this.storeName, "readwrite")
-        .put(result);
-    }
-
-    return result && result.value;
-  }
-
-  async getAll() {
-    let result = new Map();
-    try {
-      let db = await StartupCache.open();
-
-      let results = await db.objectStore(this.storeName)
-                            .getAll();
-      for (let {key, value} of results) {
-        result.set(key, value);
-      }
-    } catch (e) {
-      Cu.reportError(e);
+      result = await createFunc(path);
+      store.set(key, result);
+      StartupCache.save();
     }
 
     return result;
   }
 
-  async delete(key) {
-    let db = await StartupCache.open();
+  async getAll() {
+    let [store] = await this.getStore();
+
+    return new Map(store);
+  }
 
-    return db.objectStore(this.storeName, "readwrite").delete(key);
+  async delete(path) {
+    let [store, key] = await this.getStore(path);
+
+    store.delete(key);
+    StartupCache.save();
   }
 }
 
 for (let name of StartupCache.STORE_NAMES) {
   StartupCache[name] = new CacheStore(name);
 }
 
 /**
@@ -1069,16 +1098,17 @@ this.ExtensionUtils = {
   getWinUtils,
   ignoreEvent,
   injectAPI,
   instanceOf,
   normalizeTime,
   promiseDocumentLoaded,
   promiseDocumentReady,
   promiseEvent,
+  promiseFileContents,
   promiseObserved,
   runSafe,
   runSafeSync,
   runSafeSyncWithoutClone,
   runSafeWithoutClone,
   stylesheetMap,
   DefaultMap,
   DefaultWeakMap,