Bug 1358907 Part 3 Avoid reading XPI database at startup draft
authorAndrew Swan <aswan@mozilla.com>
Thu, 18 May 2017 13:08:58 -0700
changeset 605544 470b01c48959221799a7aa0ec4419cfa97b5dfea
parent 605543 7913e2acfdf14f490a1e0b8dd55823b8350dfeef
child 636526 557eea60814d2846a7f46866e6eabece75736d08
push id67445
push useraswan@mozilla.com
push dateFri, 07 Jul 2017 22:57:23 +0000
bugs1358907
milestone56.0a1
Bug 1358907 Part 3 Avoid reading XPI database at startup Switch telemetry and experiments from AddonManager.getAddonsByTypes() to AddonManager.getActiveAddons() which gives us less detailed information in the environment during startup but also means we don't need to load the extensions database until startup is complete. MozReview-Commit-ID: 4SxdPHSPovB
browser/experiments/Experiments.jsm
browser/experiments/test/xpcshell/head.js
toolkit/components/telemetry/TelemetryEnvironment.jsm
toolkit/components/telemetry/docs/data/environment.rst
toolkit/components/telemetry/tests/unit/head.js
toolkit/components/telemetry/tests/unit/test_ChildEvents.js
toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
toolkit/components/telemetry/tests/unit/test_ChildScalars.js
toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
toolkit/components/telemetry/tests/unit/test_TelemetryController.js
toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
toolkit/components/telemetry/tests/unit/test_TelemetrySession_activeTicks.js
toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
toolkit/mozapps/extensions/internal/XPIProvider.jsm
--- a/browser/experiments/Experiments.jsm
+++ b/browser/experiments/Experiments.jsm
@@ -159,24 +159,40 @@ function loadJSONAsync(file, options) {
 // Returns a promise that is resolved with the AddonInstall for that URL.
 function addonInstallForURL(url, hash) {
   return AddonManager.getInstallForURL(url, null, "application/x-xpinstall", hash);
 }
 
 // Returns a promise that is resolved with an Array<Addon> of the installed
 // experiment addons.
 function installedExperimentAddons() {
-  return AddonManager.getAddonsByTypes(["experiment"]).then(addons => {
+  return AddonManager.getActiveAddons(["experiment"]).then(addons => {
     return addons.filter(a => !a.appDisabled);
   });
 }
 
 // Takes an Array<Addon> and returns a promise that is resolved when the
 // addons are uninstalled.
-function uninstallAddons(addons) {
+async function uninstallAddons(addons) {
+  if (!AddonManagerPrivate.isDBLoaded()) {
+    await new Promise(resolve => {
+      Services.obs.addObserver({
+        observe(subject, topic, data) {
+          Services.obs.removeObserver(this, "xpi-database-loaded");
+          resolve();
+        },
+      }, "xpi-database-loaded");
+    });
+
+    // This function was called during startup so the addons that were
+    // passed in were partial addon objects.  Now that the full addons
+    // database is loaded, get proper Addon objects.
+    addons = await AddonManager.getAddonsByIDs(addons.map(a => a.id));
+  }
+
   let ids = new Set(addons.map(addon => addon.id));
   return new Promise(resolve => {
 
     let listener = {};
     listener.onUninstalled = addon => {
       if (!ids.has(addon.id)) {
         return;
       }
@@ -186,20 +202,16 @@ function uninstallAddons(addons) {
         AddonManager.removeAddonListener(listener);
         resolve();
       }
     };
 
     AddonManager.addAddonListener(listener);
 
     for (let addon of addons) {
-      // Disabling the add-on before uninstalling is necessary to cause tests to
-      // pass. This might be indicative of a bug in XPIProvider.
-      // TODO follow up in bug 992396.
-      addon.userDisabled = true;
       addon.uninstall();
     }
 
   });
 }
 
 /**
  * The experiments module.
--- a/browser/experiments/test/xpcshell/head.js
+++ b/browser/experiments/test/xpcshell/head.js
@@ -150,16 +150,17 @@ const {
 
 // Starts the addon manager without creating app info. We can't directly use
 // |loadAddonManager| defined above in test_conditions.js as it would make the test fail.
 function startAddonManagerOnly() {
   let addonManager = Cc["@mozilla.org/addons/integration;1"]
                        .getService(Ci.nsIObserver)
                        .QueryInterface(Ci.nsITimerCallback);
   addonManager.observe(null, "addons-startup", null);
+  Services.obs.notifyObservers(null, "sessionstore-windows-restored");
 }
 
 function getExperimentAddons(previous = false) {
   return new Promise(resolve => {
 
     AddonManager.getAddonsByTypes(["experiment"], (addons) => {
       if (previous) {
         resolve(addons);
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm
@@ -17,21 +17,22 @@ Cu.import("resource://gre/modules/Prefer
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
 Cu.import("resource://gre/modules/ObjectUtils.jsm");
 Cu.import("resource://gre/modules/TelemetryController.jsm", this);
 Cu.import("resource://gre/modules/AppConstants.jsm");
 
 const Utils = TelemetryUtils;
 
+const { AddonManager, AddonManagerPrivate } = Cu.import("resource://gre/modules/AddonManager.jsm", {});
+
 XPCOMUtils.defineLazyModuleGetter(this, "AttributionCode",
                                   "resource:///modules/AttributionCode.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ctypes",
                                   "resource://gre/modules/ctypes.jsm");
-Cu.import("resource://gre/modules/AddonManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "LightweightThemeManager",
                                   "resource://gre/modules/LightweightThemeManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ProfileAge",
                                   "resource://gre/modules/ProfileAge.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils",
                                   "resource://gre/modules/UpdateUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "WindowsRegistry",
                                   "resource://gre/modules/WindowsRegistry.jsm");
@@ -476,23 +477,44 @@ EnvironmentAddonBuilder.prototype = {
     // unfortunate reality of life.
     try {
       AddonManager.shutdown.addBlocker("EnvironmentAddonBuilder",
         () => this._shutdownBlocker());
     } catch (err) {
       return Promise.reject(err);
     }
 
-    this._pendingTask = this._updateAddons().then(
-      () => { this._pendingTask = null; },
-      (err) => {
+    this._pendingTask = (async () => {
+      try {
+        // Gather initial addons details
+        await this._updateAddons();
+
+        if (!AddonManagerPrivate.isDBLoaded()) {
+          // The addon database has not been loaded, so listen for the event
+          // triggered by the AddonManager when it is loaded so we can
+          // immediately gather full data at that time.
+          await new Promise(resolve => {
+            const ADDON_LOAD_NOTIFICATION = "xpi-database-loaded";
+            Services.obs.addObserver({
+              observe(subject, topic, data) {
+                Services.obs.removeObserver(this, ADDON_LOAD_NOTIFICATION);
+                resolve();
+              },
+            }, ADDON_LOAD_NOTIFICATION);
+          });
+
+          // Now gather complete addons details.
+          await this._updateAddons();
+        }
+      } catch (err) {
         this._environment._log.error("init - Exception in _updateAddons", err);
+      } finally {
         this._pendingTask = null;
       }
-    );
+    })();
 
     return this._pendingTask;
   },
 
   /**
    * Register an addon listener and watch for changes.
    */
   watchForChanges() {
@@ -545,16 +567,22 @@ EnvironmentAddonBuilder.prototype = {
       });
   },
 
   _shutdownBlocker() {
     if (this._loaded) {
       AddonManager.removeAddonListener(this);
       Services.obs.removeObserver(this, EXPERIMENTS_CHANGED_TOPIC);
     }
+
+    // At startup, _pendingTask is set to a Promise that does not resolve
+    // until the addons database has been read so complete details about
+    // addons are available.  Returning it here will cause it to block
+    // profileBeforeChange, guranteeing that full information will be
+    // available by the time profileBeforeChangeTelemetry is fired.
     return this._pendingTask;
   },
 
   /**
    * Collect the addon data for the environment.
    *
    * This should only be called from _pendingTask; otherwise we risk
    * running this during addon manager shutdown.
@@ -595,70 +623,69 @@ EnvironmentAddonBuilder.prototype = {
   },
 
   /**
    * Get the addon data in object form.
    * @return Promise<object> containing the addon data.
    */
   async _getActiveAddons() {
     // Request addons, asynchronously.
-    let allAddons = await AddonManager.getAddonsByTypes(["extension", "service"]);
+    let allAddons = await AddonManager.getActiveAddons(["extension", "service"]);
 
+    let isDBLoaded = AddonManagerPrivate.isDBLoaded();
     let activeAddons = {};
     for (let addon of allAddons) {
-      // Skip addons which are not active.
-      if (!addon.isActive) {
-        continue;
-      }
-
       // Weird addon data in the wild can lead to exceptions while collecting
       // the data.
       try {
         // Make sure to have valid dates.
-        let installDate = new Date(Math.max(0, addon.installDate));
         let updateDate = new Date(Math.max(0, addon.updateDate));
 
         activeAddons[addon.id] = {
-          blocklisted: (addon.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED),
-          description: limitStringToLength(addon.description, MAX_ADDON_STRING_LENGTH),
-          name: limitStringToLength(addon.name, MAX_ADDON_STRING_LENGTH),
-          userDisabled: enforceBoolean(addon.userDisabled),
-          appDisabled: addon.appDisabled,
           version: limitStringToLength(addon.version, MAX_ADDON_STRING_LENGTH),
           scope: addon.scope,
           type: addon.type,
-          foreignInstall: enforceBoolean(addon.foreignInstall),
-          hasBinaryComponents: addon.hasBinaryComponents,
-          installDay: Utils.millisecondsToDays(installDate.getTime()),
           updateDay: Utils.millisecondsToDays(updateDate.getTime()),
-          signedState: addon.signedState,
           isSystem: addon.isSystem,
           isWebExtension: addon.isWebExtension,
           multiprocessCompatible: Boolean(addon.multiprocessCompatible),
         };
 
-        if (addon.signedState !== undefined)
-          activeAddons[addon.id].signedState = addon.signedState;
-
+        // getActiveAddons() gives limited data during startup and full
+        // data after the addons database is loaded.
+        if (isDBLoaded) {
+          let installDate = new Date(Math.max(0, addon.installDate));
+          Object.assign(activeAddons[addon.id], {
+            blocklisted: (addon.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED),
+            description: limitStringToLength(addon.description, MAX_ADDON_STRING_LENGTH),
+            name: limitStringToLength(addon.name, MAX_ADDON_STRING_LENGTH),
+            userDisabled: enforceBoolean(addon.userDisabled),
+            appDisabled: addon.appDisabled,
+            foreignInstall: enforceBoolean(addon.foreignInstall),
+            hasBinaryComponents: addon.hasBinaryComponents,
+            installDay: Utils.millisecondsToDays(installDate.getTime()),
+            signedState: addon.signedState,
+          });
+        }
       } catch (ex) {
         this._environment._log.error("_getActiveAddons - An addon was discarded due to an error", ex);
         continue;
       }
     }
 
     return activeAddons;
   },
 
   /**
    * Get the currently active theme data in object form.
    * @return Promise<object> containing the active theme data.
    */
   async _getActiveTheme() {
     // Request themes, asynchronously.
-    let themes = await AddonManager.getAddonsByTypes(["theme"]);
+    let themes = await AddonManager.getActiveAddons(["theme"]);
 
     let activeTheme = {};
     // We only store information about the active theme.
     let theme = themes.find(theme => theme.isActive);
     if (theme) {
       // Make sure to have valid dates.
       let installDate = new Date(Math.max(0, theme.installDate));
       let updateDate = new Date(Math.max(0, theme.updateDate));
--- a/toolkit/components/telemetry/docs/data/environment.rst
+++ b/toolkit/components/telemetry/docs/data/environment.rst
@@ -210,17 +210,17 @@ Structure:
             description: <string>, // null if not available
             name: <string>,
             userDisabled: <bool>,
             appDisabled: <bool>,
             version: <string>,
             scope: <integer>,
             type: <string>, // "extension", "service", ...
             foreignInstall: <bool>,
-            hasBinaryComponents: <bool>
+            hasBinaryComponents: <bool>,
             installDay: <number>, // days since UNIX epoch, 0 on failure
             updateDay: <number>, // days since UNIX epoch, 0 on failure
             signedState: <integer>, // whether the add-on is signed by AMO, only present for extensions
             isSystem: <bool>, // true if this is a System Add-on
             isWebExtension: <bool>, // true if this is a WebExtension
             multiprocessCompatible: <bool>, // true if this add-on does *not* require e10s shims
           },
           ...
@@ -390,11 +390,13 @@ This object contains operating system in
 addons
 ------
 
 activeAddons
 ~~~~~~~~~~~~
 
 Starting from Firefox 44, the length of the following string fields: ``name``, ``description`` and ``version`` is limited to 100 characters. The same limitation applies to the same fields in ``theme`` and ``activePlugins``.
 
+Some of the fields in the record for each addon are not available during startup.  The fields that will always be present are ``id``, ``version``, ``type``, ``updateDate``, ``scope``, ``isSystem``, ``isWebExtension``, and ``multiprocessCompatible``.  All the other fields documented above become present shortly after the ``sessionstore-windows-restored`` event is dispatched.
+
 experiments
 -----------
 For each experiment we collect the ``id`` and the ``branch`` the client is enrolled in. Both fields are truncated to 100 characters and a warning is printed when that happens. This section will eventually supersede ``addons/activeExperiment``.
--- a/toolkit/components/telemetry/tests/unit/head.js
+++ b/toolkit/components/telemetry/tests/unit/head.js
@@ -173,16 +173,20 @@ function loadAddonManager(...args) {
 
   // As we're not running in application, we need to setup the features directory
   // used by system add-ons.
   const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "app0"], true);
   AddonTestUtils.registerDirectory("XREAppFeat", distroDir);
   return AddonTestUtils.promiseStartupManager();
 }
 
+function finishAddonManagerStartup() {
+  Services.obs.notifyObservers(null, "test-load-xpi-database");
+}
+
 var gAppInfo = null;
 
 function createAppInfo(ID = "xpcshell@tests.mozilla.org", name = "XPCShell",
                        version = "1.0", platformVersion = "1.0") {
   AddonTestUtils.createAppInfo(ID, name, version, platformVersion);
   gAppInfo = AddonTestUtils.appInfo;
 }
 
--- a/toolkit/components/telemetry/tests/unit/test_ChildEvents.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildEvents.js
@@ -61,16 +61,17 @@ add_task(async function() {
     run_child_test();
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
+  finishAddonManagerStartup();
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   await TelemetryController.testSetup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
   // Enable recording for the test event category.
   Telemetry.setEventRecordingEnabled("telemetry.test", true);
 
   // Run test in child, don't wait for it to finish: just wait for the
--- a/toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
@@ -87,16 +87,17 @@ add_task(async function() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
+  finishAddonManagerStartup();
   await TelemetryController.testSetup();
   if (runningInParent) {
     // Make sure we don't generate unexpected pings due to pref changes.
     await setEmptyPrefWatchlist();
   }
 
   // Run test in child, don't wait for it to finish.
   run_test_in_child("test_ChildHistograms.js");
--- a/toolkit/components/telemetry/tests/unit/test_ChildScalars.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildScalars.js
@@ -136,16 +136,17 @@ add_task(async function() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
+  finishAddonManagerStartup();
   await TelemetryController.testSetup();
   if (runningInParent) {
     setParentScalars();
     // Make sure we don't generate unexpected pings due to pref changes.
     await setEmptyPrefWatchlist();
   }
 
   // Run test in child, don't wait for it to finish: just wait for the
--- a/toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
+++ b/toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
@@ -82,16 +82,17 @@ var promiseValidateArchivedPings = async
 };
 
 add_task(async function test_setup() {
   do_test_pending();
 
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Preferences.set(TelemetryUtils.Preferences.TelemetryEnabled, true);
 });
 
 add_task(async function test_subsessionsChaining() {
   if (gIsAndroid) {
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
@@ -87,16 +87,17 @@ function checkPingFormat(aPing, aType, a
   Assert.equal("clientId" in aPing, aHasClientId);
   Assert.equal("environment" in aPing, aHasEnvironment);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 
   await new Promise(resolve =>
     Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(resolve)));
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
@@ -21,16 +21,17 @@ function contentHandler(metadata, respon
   response.processAsync();
   response.setHeader("Content-Type", "text/plain");
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 });
 
 /**
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
@@ -1,15 +1,17 @@
 /* Any copyright is dedicated to the Public Domain.
  * http://creativecommons.org/publicdomain/zero/1.0/ */
 
-Cu.import("resource://gre/modules/AddonManager.jsm");
+const {AddonManager, AddonManagerPrivate} = Cu.import("resource://gre/modules/AddonManager.jsm", {});
 Cu.import("resource://gre/modules/TelemetryEnvironment.jsm", this);
+Cu.import("resource://gre/modules/ObjectUtils.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm", this);
 Cu.import("resource://gre/modules/PromiseUtils.jsm", this);
+Cu.import("resource://gre/modules/Timer.jsm", this);
 Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
 Cu.import("resource://testing-common/AddonManagerTesting.jsm");
 Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://testing-common/MockRegistrar.jsm", this);
 Cu.import("resource://gre/modules/FileUtils.jsm");
 
 // AttributionCode is only needed for Firefox
 XPCOMUtils.defineLazyModuleGetter(this, "AttributionCode",
@@ -652,48 +654,58 @@ function checkSystemSection(data) {
     let features = gfxInfo.getFeatures();
     Assert.equal(features.compositor, gfxData.features.compositor);
     Assert.equal(features.gpuProcess.status, gfxData.features.gpuProcess.status);
     Assert.equal(features.opengl, gfxData.features.opengl);
     Assert.equal(features.webgl, gfxData.features.webgl);
   } catch (e) {}
 }
 
-function checkActiveAddon(data) {
+function checkActiveAddon(data, partialRecord) {
   let signedState = mozinfo.addon_signing ? "number" : "undefined";
   // system add-ons have an undefined signState
   if (data.isSystem)
     signedState = "undefined";
 
   const EXPECTED_ADDON_FIELDS_TYPES = {
-    blocklisted: "boolean",
-    name: "string",
-    userDisabled: "boolean",
-    appDisabled: "boolean",
     version: "string",
     scope: "number",
     type: "string",
-    foreignInstall: "boolean",
-    hasBinaryComponents: "boolean",
-    installDay: "number",
     updateDay: "number",
-    signedState,
     isSystem: "boolean",
     isWebExtension: "boolean",
     multiprocessCompatible: "boolean",
   };
 
-  for (let f in EXPECTED_ADDON_FIELDS_TYPES) {
-    Assert.ok(f in data, f + " must be available.");
-    Assert.equal(typeof data[f], EXPECTED_ADDON_FIELDS_TYPES[f],
-                 f + " must have the correct type.");
+  const FULL_ADDON_FIELD_TYPES = {
+    blocklisted: "boolean",
+    name: "string",
+    userDisabled: "boolean",
+    appDisabled: "boolean",
+    foreignInstall: "boolean",
+    hasBinaryComponents: "boolean",
+    installDay: "number",
+    signedState,
+  };
+
+  let fields = EXPECTED_ADDON_FIELDS_TYPES;
+  if (!partialRecord) {
+    fields = Object.assign({}, fields, FULL_ADDON_FIELD_TYPES);
   }
 
-  // We check "description" separately, as it can be null.
-  Assert.ok(checkNullOrString(data.description));
+  for (let [name, type] of Object.entries(fields)) {
+    Assert.ok(name in data, name + " must be available.");
+    Assert.equal(typeof data[name], type,
+                 name + " must have the correct type.");
+  }
+
+  if (!partialRecord) {
+    // We check "description" separately, as it can be null.
+    Assert.ok(checkNullOrString(data.description));
+  }
 }
 
 function checkPlugin(data) {
   const EXPECTED_PLUGIN_FIELDS_TYPES = {
     name: "string",
     version: "string",
     description: "string",
     blocklisted: "boolean",
@@ -743,32 +755,32 @@ function checkActiveGMPlugin(data) {
   // GMP plugin version defaults to null until GMPDownloader runs to update it.
   if (data.version) {
     Assert.equal(typeof data.version, "string");
   }
   Assert.equal(typeof data.userDisabled, "boolean");
   Assert.equal(typeof data.applyBackgroundUpdates, "number");
 }
 
-function checkAddonsSection(data, expectBrokenAddons) {
+function checkAddonsSection(data, expectBrokenAddons, partialAddonsRecords) {
   const EXPECTED_FIELDS = [
     "activeAddons", "theme", "activePlugins", "activeGMPlugins", "activeExperiment",
     "persona",
   ];
 
   Assert.ok("addons" in data, "There must be an addons section in Environment.");
   for (let f of EXPECTED_FIELDS) {
     Assert.ok(f in data.addons, f + " must be available.");
   }
 
   // Check the active addons, if available.
   if (!expectBrokenAddons) {
     let activeAddons = data.addons.activeAddons;
     for (let addon in activeAddons) {
-      checkActiveAddon(activeAddons[addon]);
+      checkActiveAddon(activeAddons[addon], partialAddonsRecords);
     }
   }
 
   // Check "theme" structure.
   if (Object.keys(data.addons.theme).length !== 0) {
     checkTheme(data.addons.theme);
   }
 
@@ -810,17 +822,22 @@ function checkExperimentsSection(data) {
     Assert.ok("branch" in experimentData, "The experiment must have branch data.")
     Assert.ok(checkString(experimentData.branch), "The experiment data must be valid.");
     if ("type" in experimentData) {
       Assert.ok(checkString(experimentData.type));
     }
   }
 }
 
-function checkEnvironmentData(data, isInitial = false, expectBrokenAddons = false) {
+function checkEnvironmentData(data, options = {}) {
+  const {
+    isInitial = false,
+    expectBrokenAddons = false,
+  } = options;
+
   checkBuildSection(data);
   checkSettingsSection(data);
   checkProfileSection(data);
   checkPartnerSection(data, isInitial);
   checkSystemSection(data);
   checkAddonsSection(data, expectBrokenAddons);
   checkExperimentsSection(data);
 }
@@ -838,16 +855,24 @@ add_task(async function setup() {
   let system_addon = FileUtils.File(distroDir.path);
   system_addon.append("tel-system-xpi@tests.mozilla.org.xpi");
   system_addon.lastModifiedTime = SYSTEM_ADDON_INSTALL_DATE;
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
 
   // Spoof the persona ID.
   LightweightThemeManager.currentTheme =
     spoofTheme(PERSONA_ID, PERSONA_NAME, PERSONA_DESCRIPTION);
+
+  // The test runs in a fresh profile so starting the AddonManager causes
+  // the addons database to be created (as does setting new theme).
+  // For test_addonsStartup below, we want to test a "warm" startup where
+  // there is already a database on disk.  Simulate that here by just
+  // restarting the AddonManager.
+  await AddonTestUtils.promiseRestartManager();
+
   // Register a fake plugin host for consistent flash version data.
   registerFakePluginHost();
 
   // Setup a webserver to serve Addons, Plugins, etc.
   gHttpServer = new HttpServer();
   gHttpServer.start(-1);
   let port = gHttpServer.identity.primaryPort;
   gHttpRoot = "http://localhost:" + port + "/";
@@ -868,18 +893,29 @@ add_task(async function setup() {
     do_register_cleanup(cleanupAttributionData);
   }
 
   await spoofProfileReset();
   TelemetryEnvironment.delayedInit();
 });
 
 add_task(async function test_checkEnvironment() {
-  let environmentData = await TelemetryEnvironment.onInitialized();
-  checkEnvironmentData(environmentData, true);
+  // During startup we have partial addon records.
+  // First make sure we haven't yet read the addons DB, then test that
+  // we have some partial addons data.
+  Assert.equal(AddonManagerPrivate.isDBLoaded(), false,
+               "addons database is not loaded");
+
+  checkAddonsSection(TelemetryEnvironment.currentEnvironment, false, true);
+
+  // Now continue with startup.
+  let initPromise = TelemetryEnvironment.onInitialized();
+  finishAddonManagerStartup();
+  let environmentData = await initPromise;
+  checkEnvironmentData(environmentData, {isInitial: true});
 
   spoofPartnerInfo();
   Services.obs.notifyObservers(null, DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC);
 
   environmentData = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(environmentData);
 });
 
@@ -1407,17 +1443,17 @@ add_task(async function test_collectionW
   checkpointPromise = registerCheckpointPromise(2);
   await AddonManagerTesting.installXPIFromURL(ADDON_INSTALL_URL);
   await checkpointPromise;
   assertCheckpoint(2);
 
   // Check that the new environment contains the Social addon installed with the broken
   // manifest and the rest of the data.
   let data = TelemetryEnvironment.currentEnvironment;
-  checkEnvironmentData(data, false, true /* expect broken addons*/);
+  checkEnvironmentData(data, {expectBrokenAddons: true});
 
   let activeAddons = data.addons.activeAddons;
   Assert.ok(BROKEN_ADDON_ID in activeAddons,
             "The addon with the broken manifest must be reported.");
   Assert.equal(activeAddons[BROKEN_ADDON_ID].version, null,
                "null should be reported for invalid data.");
   Assert.ok(ADDON_ID in activeAddons,
             "The valid addon must be reported.");
@@ -1429,17 +1465,18 @@ add_task(async function test_collectionW
 
   // Uninstall the valid addon.
   await AddonManagerTesting.uninstallAddonByID(ADDON_ID);
 });
 
 add_task(async function test_defaultSearchEngine() {
   // Check that no default engine is in the environment before the search service is
   // initialized.
-  let data = TelemetryEnvironment.currentEnvironment;
+
+  let data = await TelemetryEnvironment.testCleanRestart().onInitialized();
   checkEnvironmentData(data);
   Assert.ok(!("defaultSearchEngine" in data.settings));
   Assert.ok(!("defaultSearchEngineData" in data.settings));
 
   // Load the engines definitions from a custom JAR file: that's needed so that
   // the search provider reports an engine identifier.
   let url = "chrome://testsearchplugin/locale/searchplugins/";
   let resProt = Services.io.getProtocolHandler("resource")
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
@@ -47,16 +47,17 @@ function setMinimumPolicyVersion(aNewPol
   // We don't have a channel specific minimu, so set the common one.
   Preferences.set(TelemetryUtils.Preferences.MinimumPolicyVersion, aNewPolicyVersion);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile(true);
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
 
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   // Don't bypass the notifications in this test, we'll fake it.
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.BypassNotification, false);
 
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
@@ -139,16 +139,17 @@ function pingHandler(aRequest) {
   gSeenPings++;
 }
 
 add_task(async function test_setup() {
   PingServer.start();
   PingServer.registerPingHandler(pingHandler);
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   Services.prefs.setCharPref(TelemetryUtils.Preferences.Server,
                               "http://localhost:" + PingServer.port);
 });
 
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
@@ -468,16 +468,17 @@ function write_fake_failedprofilelocks_f
   let contents = "" + FAILED_PROFILE_LOCK_ATTEMPTS;
   writeStringToFile(file, contents);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 
   // Make it look like we've previously failed to lock a profile a couple times.
   write_fake_failedprofilelocks_file();
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySession_activeTicks.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySession_activeTicks.js
@@ -5,16 +5,17 @@
 Cu.import("resource://gre/modules/TelemetryController.jsm", this);
 Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
 
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  finishAddonManagerStartup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.TelemetryEnabled, true);
 });
 
 add_task(async function test_record_activeTicks() {
   await TelemetryController.testSetup();
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
@@ -19,16 +19,17 @@ var gGlobalScope = this;
 
 function getSimpleMeasurementsFromTelemetryController() {
   return TelemetrySession.getPayload().simpleMeasurements;
 }
 
 add_task(async function test_setup() {
   // Telemetry needs the AddonManager.
   loadAddonManager();
+  finishAddonManagerStartup();
   // Make profile available for |TelemetryController.testShutdown()|.
   do_get_profile();
 
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   await new Promise(resolve =>
     Services.telemetry.asyncFetchTelemetryData(resolve));
--- a/toolkit/mozapps/extensions/internal/XPIProvider.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIProvider.jsm
@@ -2265,30 +2265,51 @@ this.XPIProvider = {
       Services.obs.addObserver({
         observe(aSubject, aTopic, aData) {
           AddonManagerPrivate.recordTimestamp("XPI_finalUIStartup");
           XPIProvider.runPhase = XPI_AFTER_UI_STARTUP;
           Services.obs.removeObserver(this, "final-ui-startup");
         }
       }, "final-ui-startup");
 
-      // Once other important startup work is finished, try to load the
-      // XPI database so that the telemetry environment can be populated
-      // with detailed addon information.
+      // If we haven't yet loaded the XPI database, schedule loading it
+      // to occur once other important startup work is finished.  We want
+      // this to happen relatively quickly after startup so the telemetry
+      // environment has complete addon information.
+      //
+      // Unfortunately we have to use a variety of ways do detect when it
+      // is time to load.  In a regular browser process we just wait for
+      // sessionstore-windows-restored.  In a browser toolbox process
+      // we wait for the toolbox to show up, based on xul-window-visible
+      // and a visible toolbox window.
+      // Finally, we have a test-only event called test-load-xpi-database
+      // as a temporary workaround for bug 1372845.  The latter can be
+      // cleaned up when that bug is resolved.
       if (!this.isDBLoaded) {
-        Services.obs.addObserver({
+        const EVENTS = [ "sessionstore-windows-restored", "xul-window-visible", "test-load-xpi-database" ];
+        let observer = {
           observe(subject, topic, data) {
-            Services.obs.removeObserver(this, "sessionstore-windows-restored");
+            if (topic == "xul-window-visible" &&
+                !Services.wm.getMostRecentWindow("devtools:toolbox")) {
+              return;
+            }
+
+            for (let event of EVENTS) {
+              Services.obs.removeObserver(observer, event);
+            }
 
             // It would be nice to defer some of the work here until we
             // have idle time but we can't yet use requestIdleCallback()
             // from chrome.  See bug 1358476.
             XPIDatabase.asyncLoadDB();
           },
-        }, "sessionstore-windows-restored");
+        };
+        for (let event of EVENTS) {
+          Services.obs.addObserver(observer, event);
+        }
       }
 
       AddonManagerPrivate.recordTimestamp("XPI_startup_end");
 
       this.extensionsActive = true;
       this.runPhase = XPI_BEFORE_UI_STARTUP;
 
       let timerManager = Cc["@mozilla.org/updates/timer-manager;1"].