Bug 1475306 - Changes to the detection of extensions already migrated to ExtensionStorageIDB and rename successfully migrated JSONFile. r=mixedpuppy draft
authorLuca Greco <lgreco@mozilla.com>
Fri, 06 Jul 2018 19:07:28 +0200
changeset 819764 11c3e291ac8323a679c2bb29efc7df3700023591
parent 818682 2ed1506d1dc7db3d70a3feed95f1456bce05bbee
push id116651
push userluca.greco@alcacoop.it
push dateWed, 18 Jul 2018 15:46:34 +0000
reviewersmixedpuppy
bugs1475306
milestone63.0a1
Bug 1475306 - Changes to the detection of extensions already migrated to ExtensionStorageIDB and rename successfully migrated JSONFile. r=mixedpuppy This patch applies the following changes to the storage.local data migration behaviors: - An about:config preference is set when an extension has been migrated successfully to the storage.local IndexedDB backend (cleared automatically if the addon is uninstalled). - If the above about:config preference is set, the storage.local IndexedDB backend is enabled without attempting to open an IndexedDB connection for the new backend. - While migrating an extension, if we fail to open the IndexedDB connection, the data migration is cancelled and the storage.local API is going to fallback to the storage.local JSONFile backend (until the next extension startup, when a new data migration is going to be tried). - When a migration is completed successfully, the old JSONFile is renamed (by appending ".migrated" to its original file name) instead of being removed. MozReview-Commit-ID: LPM0fQUagTd
toolkit/components/extensions/Extension.jsm
toolkit/components/extensions/ExtensionStorage.jsm
toolkit/components/extensions/ExtensionStorageIDB.jsm
toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js
toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js
--- a/toolkit/components/extensions/Extension.jsm
+++ b/toolkit/components/extensions/Extension.jsm
@@ -142,16 +142,17 @@ function classifyPermission(perm) {
   }
   return {permission: perm};
 }
 
 const LOGGER_ID_BASE = "addons.webextension.";
 const UUID_MAP_PREF = "extensions.webextensions.uuids";
 const LEAVE_STORAGE_PREF = "extensions.webextensions.keepStorageOnUninstall";
 const LEAVE_UUID_PREF = "extensions.webextensions.keepUuidOnUninstall";
+const IDB_MIGRATED_PREF_BRANCH = "extensions.webextensions.ExtensionStorageIDB.migrated";
 
 const COMMENT_REGEXP = new RegExp(String.raw`
     ^
     (
       (?:
         [^"\n] |
         " (?:[^"\\\n] | \\.)* "
       )*?
@@ -246,16 +247,19 @@ var UninstallObserver = {
       Services.qms.clearStoragesForPrincipal(principal);
 
       // Clear any storage.local data stored in the IDBBackend.
       let storagePrincipal = Services.scriptSecurityManager.createCodebasePrincipal(baseURI, {
         userContextId: WEBEXT_STORAGE_USER_CONTEXT_ID,
       });
       Services.qms.clearStoragesForPrincipal(storagePrincipal);
 
+      // Clear the preference set for the extensions migrated to the IDBBackend.
+      Services.prefs.clearUserPref(`${IDB_MIGRATED_PREF_BRANCH}.${addon.id}`);
+
       // Clear localStorage created by the extension
       let storage = Services.domStorageManager.getStorage(null, principal);
       if (storage) {
         storage.clear();
       }
 
       // Remove any permissions related to the unlimitedStorage permission
       // if we are also removing all the data stored by the extension.
--- a/toolkit/components/extensions/ExtensionStorage.jsm
+++ b/toolkit/components/extensions/ExtensionStorage.jsm
@@ -132,18 +132,22 @@ var ExtensionStorage = {
   /**
    * Clear the cached jsonFilePromise for a given extensionId
    * (used by ExtensionStorageIDB to free the jsonFile once the data migration
    * has been completed).
    *
    * @param {string} extensionId
    *        The ID of the extension for which to return a file.
    */
-  clearCachedFile(extensionId) {
-    this.jsonFilePromises.delete(extensionId);
+  async clearCachedFile(extensionId) {
+    let promise = this.jsonFilePromises.get(extensionId);
+    if (promise) {
+      this.jsonFilePromises.delete(extensionId);
+      await promise.then(jsonFile => jsonFile.finalize());
+    }
   },
 
   /**
    * Sanitizes the given value, and returns a JSON-compatible
    * representation of it, based on the privileges of the given global.
    *
    * @param {value} value
    *        The value to sanitize.
--- a/toolkit/components/extensions/ExtensionStorageIDB.jsm
+++ b/toolkit/components/extensions/ExtensionStorageIDB.jsm
@@ -26,16 +26,17 @@ XPCOMUtils.defineLazyGetter(this, "WEBEX
 
 const IDB_NAME = "webExtensions-storage-local";
 const IDB_DATA_STORENAME = "storage-local-data";
 const IDB_VERSION = 1;
 const IDB_MIGRATE_RESULT_HISTOGRAM = "WEBEXT_STORAGE_LOCAL_IDB_MIGRATE_RESULT_COUNT";
 
 // Whether or not the installed extensions should be migrated to the storage.local IndexedDB backend.
 const BACKEND_ENABLED_PREF = "extensions.webextensions.ExtensionStorageIDB.enabled";
+const IDB_MIGRATED_PREF_BRANCH = "extensions.webextensions.ExtensionStorageIDB.migrated";
 
 class ExtensionStorageLocalIDB extends IndexedDB {
   onupgradeneeded(event) {
     if (event.oldVersion < 1) {
       this.createObjectStore(IDB_DATA_STORENAME);
     }
   }
 
@@ -232,93 +233,123 @@ class ExtensionStorageLocalIDB extends I
  *        the ExtensionStorageLocalIDB instance.
  */
 async function migrateJSONFileData(extension, storagePrincipal) {
   let oldStoragePath;
   let oldStorageExists;
   let idbConn;
   let jsonFile;
   let hasEmptyIDB;
-  let oldDataRead = false;
-  let migrated = false;
   let histogram = Services.telemetry.getHistogramById(IDB_MIGRATE_RESULT_HISTOGRAM);
+  let dataMigrateCompleted = false;
+
+  const isMigratedExtension = Services.prefs.getBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${extension.id}`, false);
+  if (isMigratedExtension) {
+    return;
+  }
 
   try {
     idbConn = await ExtensionStorageLocalIDB.openForPrincipal(storagePrincipal);
     hasEmptyIDB = await idbConn.isEmpty();
 
     if (!hasEmptyIDB) {
       // If the IDB backend is enabled and there is data already stored in the IDB backend,
       // there is no "going back": any data that has not been migrated will be still on disk
       // but it is not going to be migrated anymore, it could be eventually used to allow
       // a user to manually retrieve the old data file).
+      Services.prefs.setBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${extension.id}`, true);
       return;
     }
+  } catch (err) {
+    extension.logWarning(
+      `storage.local data migration cancelled, unable to open IDB connection: ${err.message}::${err.stack}`);
+
+    histogram.add("failure");
+
+    throw err;
+  }
+
+  try {
+    oldStoragePath = ExtensionStorage.getStorageFile(extension.id);
+    oldStorageExists = await OS.File.exists(oldStoragePath).catch(fileErr => {
+      // If we can't access the oldStoragePath here, then extension is also going to be unable to
+      // access it, and so we log the error but we don't stop the extension from switching to
+      // the IndexedDB backend.
+      extension.logWarning(
+        `Unable to access extension storage.local data file: ${fileErr.message}::${fileErr.stack}`);
+      return false;
+    });
 
     // Migrate any data stored in the JSONFile backend (if any), and remove the old data file
     // if the migration has been completed successfully.
-    oldStoragePath = ExtensionStorage.getStorageFile(extension.id);
-    oldStorageExists = await OS.File.exists(oldStoragePath);
-
     if (oldStorageExists) {
       Services.console.logStringMessage(
         `Migrating storage.local data for ${extension.policy.debugName}...`);
 
       jsonFile = await ExtensionStorage.getFile(extension.id);
       const data = {};
       for (let [key, value] of jsonFile.data.entries()) {
         data[key] = value;
       }
-      oldDataRead = true;
+
       await idbConn.set(data);
-      migrated = true;
       Services.console.logStringMessage(
         `storage.local data successfully migrated to IDB Backend for ${extension.policy.debugName}.`);
     }
+
+    dataMigrateCompleted = true;
   } catch (err) {
-    extension.logWarning(`Error on migrating storage.local data: ${err.message}::${err.stack}`);
-    if (oldDataRead) {
-      // If the data has been read successfully and it has been failed to be stored
-      // into the IndexedDB backend, then clear any partially stored data and reject
+    extension.logWarning(`Error on migrating storage.local data file: ${err.message}::${err.stack}`);
+
+    if (oldStorageExists && !dataMigrateCompleted) {
+      // If the data failed to be stored into the IndexedDB backend, then we clear the IndexedDB
+      // backend to allow the extension to retry the migration on its next startup, and reject
       // the data migration promise explicitly (which would prevent the new backend
       // from being enabled for this session).
       Services.qms.clearStoragesForPrincipal(storagePrincipal);
 
       histogram.add("failure");
 
       throw err;
     }
   } finally {
-    // Finalize the jsonFile and clear the jsonFilePromise cached by the ExtensionStorage
-    // (so that the file can be immediatelly removed when we call OS.File.remove).
-    ExtensionStorage.clearCachedFile(extension.id);
-    if (jsonFile) {
-      jsonFile.finalize();
-    }
+    // Clear the jsonFilePromise cached by the ExtensionStorage.
+    await ExtensionStorage.clearCachedFile(extension.id).catch(err => {
+      extension.logWarning(err.message);
+    });
   }
 
   histogram.add("success");
 
-  // If the IDB backend has been enabled, try to remove the old storage.local data file,
-  // but keep using the selected backend even if it fails to be removed.
-  if (oldStorageExists && migrated) {
+  // If the IDB backend has been enabled, rename the old storage.local data file, but
+  // do not prevent the extension from switching to the IndexedDB backend if it fails.
+  if (oldStorageExists && dataMigrateCompleted) {
     try {
-      await OS.File.remove(oldStoragePath);
+      // Only migrate the file when it actually exists (e.g. the file name is not going to exist
+      // when it is corrupted, because JSONFile internally rename it to `.corrupt`.
+      if (await OS.File.exists(oldStoragePath)) {
+        let openInfo = await OS.File.openUnique(`${oldStoragePath}.migrated`, {humanReadable: true});
+        await openInfo.file.close();
+        await OS.File.move(oldStoragePath, openInfo.path);
+      }
     } catch (err) {
       extension.logWarning(err.message);
     }
   }
+
+  Services.prefs.setBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${extension.id}`, true);
 }
 
 /**
  * This ExtensionStorage class implements a backend for the storage.local API which
  * uses IndexedDB to store the data.
  */
 this.ExtensionStorageIDB = {
   BACKEND_ENABLED_PREF,
+  IDB_MIGRATED_PREF_BRANCH,
   IDB_MIGRATE_RESULT_HISTOGRAM,
 
   // Map<extension-id, Set<Function>>
   listeners: new Map(),
 
   // Keep track if the IDB backend has been selected or not for a running extension
   // (the selected backend should never change while the extension is running, even if the
   // related preference has been changed in the meantime):
--- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js
@@ -8,17 +8,30 @@
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 ChromeUtils.import("resource://gre/modules/ExtensionStorage.jsm");
 ChromeUtils.import("resource://gre/modules/ExtensionStorageIDB.jsm");
 
 XPCOMUtils.defineLazyModuleGetters(this, {
   OS: "resource://gre/modules/osfile.jsm",
 });
 
-const {IDB_MIGRATE_RESULT_HISTOGRAM} = ExtensionStorageIDB;
+const {
+  createAppInfo,
+  promiseShutdownManager,
+  promiseStartupManager,
+} = AddonTestUtils;
+
+AddonTestUtils.init(this);
+
+createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "42");
+
+const {
+  IDB_MIGRATED_PREF_BRANCH,
+  IDB_MIGRATE_RESULT_HISTOGRAM,
+} = ExtensionStorageIDB;
 const CATEGORIES = ["success", "failure"];
 
 async function createExtensionJSONFileWithData(extensionId, data) {
   await ExtensionStorage.set(extensionId, data);
   const jsonFile = await ExtensionStorage.getFile(extensionId);
   await jsonFile._save();
   const oldStorageFilename = ExtensionStorage.getStorageFile(extensionId);
   equal(await OS.File.exists(oldStorageFilename), true, "The old json file has been created");
@@ -37,20 +50,23 @@ function assertMigrationHistogramCount(c
   const histogram = Services.telemetry.getHistogramById(IDB_MIGRATE_RESULT_HISTOGRAM);
 
   equal(histogram.snapshot().counts[CATEGORIES.indexOf(category)], expectedCount,
         `Got the expected count on category "${category}" for histogram ${IDB_MIGRATE_RESULT_HISTOGRAM}`);
 }
 
 add_task(async function setup() {
   Services.prefs.setBoolPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF, true);
+  setLowDiskMode(false);
+
+  await promiseStartupManager();
 });
 
 // Test that the old data is migrated successfully to the new storage backend
-// and that the original JSONFile is being removed.
+// and that the original JSONFile has been renamed.
 add_task(async function test_storage_local_data_migration() {
   const EXTENSION_ID = "extension-to-be-migrated@mozilla.org";
 
   const data = {
     "test_key_string": "test_value1",
     "test_key_number": 1000,
     "test_nested_data": {
       "nested_key": true,
@@ -71,16 +87,17 @@ add_task(async function test_storage_loc
                           "Got the expected data after the storage.local data migration");
 
     browser.test.sendMessage("storage-local-data-migrated");
   }
 
   clearMigrationHistogram();
 
   let extension = ExtensionTestUtils.loadExtension({
+    useAddonManager: "temporary",
     manifest: {
       permissions: ["storage"],
       applications: {
         gecko: {
           id: EXTENSION_ID,
         },
       },
     },
@@ -94,22 +111,32 @@ add_task(async function test_storage_loc
   const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal(extension.extension);
 
   const idbConn = await ExtensionStorageIDB.open(storagePrincipal);
 
   equal(await idbConn.isEmpty(extension.extension), false,
         "Data stored in the ExtensionStorageIDB backend as expected");
 
   equal(await OS.File.exists(oldStorageFilename), false,
-        "The old json storage file should have been removed");
+        "The old json storage file name should not exist anymore");
+
+  equal(await OS.File.exists(`${oldStorageFilename}.migrated`), true,
+        "The old json storage file name should have been renamed as .migrated");
+
+  equal(Services.prefs.getBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, false),
+        true, `Got the ${IDB_MIGRATED_PREF_BRANCH} preference set to true as expected`);
 
   assertMigrationHistogramCount("success", 1);
   assertMigrationHistogramCount("failure", 0);
 
   await extension.unload();
+
+  equal(Services.prefs.getPrefType(`${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`),
+        Services.prefs.PREF_INVALID,
+        `Got the ${IDB_MIGRATED_PREF_BRANCH} preference has been cleared on addon uninstall`);
 });
 
 // Test that if the old JSONFile data file is corrupted and the old data
 // can't be successfully migrated to the new storage backend, then:
 // - the new storage backend for that extension is still initialized and enabled
 // - any new data is being stored in the new backend
 // - the old file is being renamed (with the `.corrupted` suffix that JSONFile.jsm
 //   adds when it fails to load the data file) and still available on disk.
@@ -164,17 +191,71 @@ add_task(async function test_storage_loc
   equal(await idbConn.isEmpty(extension.extension), false,
         "Data stored in the ExtensionStorageIDB backend as expected");
 
   equal(await OS.File.exists(`${oldStorageFilename}.corrupt`), true,
         "The old json storage should still be available if failed to be read");
 
   // The extension is still migrated successfully to the new backend if the file from the
   // original json file was corrupted.
+
+  equal(Services.prefs.getBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, false),
+        true, `Got the ${IDB_MIGRATED_PREF_BRANCH} preference set to true as expected`);
+
   assertMigrationHistogramCount("success", 1);
   assertMigrationHistogramCount("failure", 0);
 
   await extension.unload();
 });
 
-add_task(function test_storage_local_data_migration_clear_pref() {
+// Test that if the data migration fails because of a QuotaExceededError raised when creating the
+// storage into the IndexedDB backend, the extension does not migrate to the new backend if
+// there was a JSONFile to migrate.
+add_task(async function test_storage_local_data_migration_quota_exceeded_error() {
+  const EXTENSION_ID = "extension-quota-exceeded-error@mozilla.org";
+  const data = {"test_key_string": "test_value"};
+
+  // Set the low disk mode to force the quota manager to raise a QuotaExceededError.
+  setLowDiskMode(true);
+
+  // Store some fake data in the storage.local file backend before starting the extension.
+  await createExtensionJSONFileWithData(EXTENSION_ID, data);
+
+  async function background() {
+    const result = await browser.storage.local.get("test_key_string");
+    browser.test.assertEq("test_value", result.test_key_string,
+                          "Got the expected storage.local.get result");
+
+    browser.test.sendMessage("storage-local-quota-exceeded");
+  }
+
+  clearMigrationHistogram();
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      permissions: ["storage"],
+      applications: {
+        gecko: {
+          id: EXTENSION_ID,
+        },
+      },
+    },
+    background,
+  });
+
+  await extension.startup();
+
+  await extension.awaitMessage("storage-local-quota-exceeded");
+
+  equal(Services.prefs.getBoolPref(`${IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, false),
+        false, `Got ${IDB_MIGRATED_PREF_BRANCH} preference set to false as expected`);
+
+  await extension.unload();
+
+  assertMigrationHistogramCount("success", 0);
+  assertMigrationHistogramCount("failure", 1);
+});
+
+add_task(async function test_storage_local_data_migration_clear_pref() {
   Services.prefs.clearUserPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF);
+  setLowDiskMode(false);
+  await promiseShutdownManager();
 });
--- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js
@@ -177,16 +177,18 @@ add_task(async function test_storage_loc
 });
 
 add_task(async function test_storage_local_idb_backend_destroyed_context_promise() {
   return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]],
                       test_storage_local_call_from_destroying_context);
 });
 
 add_task(async function test_storage_local_should_not_cache_idb_open_rejections() {
+  const EXTENSION_ID = "@an-already-migrated-extension";
+
   async function test_storage_local_on_idb_disk_full_rejection() {
     let extension = ExtensionTestUtils.loadExtension({
       async background() {
         browser.test.sendMessage("ext-page-url", browser.runtime.getURL("tab.html"));
       },
       files: {
         "tab.html": `<!DOCTYPE html>
         <html>
@@ -226,16 +228,21 @@ add_task(async function test_storage_loc
             }
           });
 
           browser.test.sendMessage("extension-tab-ready");
         },
       },
       manifest: {
         permissions: ["storage"],
+        applications: {
+          gecko: {
+            id: EXTENSION_ID,
+          },
+        },
       },
     });
 
     await extension.startup();
     const url = await extension.awaitMessage("ext-page-url");
 
     let contentPage = await ExtensionTestUtils.loadContentPage(url, {extension});
     await extension.awaitMessage("extension-tab-ready");
@@ -254,12 +261,17 @@ add_task(async function test_storage_loc
     info(`Wait the storage.local API calls to resolve successfully once the disk is free again`);
     await extension.awaitMessage("storage-local-get-resolved");
     info("storage.local.set and storage.local.get resolve successfully once the disk is free again");
 
     contentPage.close();
     await extension.unload();
   }
 
-  return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]],
-                      test_storage_local_on_idb_disk_full_rejection);
+  return runWithPrefs([
+    [ExtensionStorageIDB.BACKEND_ENABLED_PREF, true],
+    // Set the migrated preference for the test extension to prevent the extension
+    // from falling back to the JSONFile storage because of an QuotaExceededError
+    // raised while migrating to the IndexedDB backend.
+    [`${ExtensionStorageIDB.IDB_MIGRATED_PREF_BRANCH}.${EXTENSION_ID}`, true],
+  ], test_storage_local_on_idb_disk_full_rejection);
 });