Bug 1328974: Record metrics for chrome.storage.sync usage, r?kmag, bsmedberg
MozReview-Commit-ID: 7c2CHLuXxS6
--- a/services/common/kinto-storage-adapter.js
+++ b/services/common/kinto-storage-adapter.js
@@ -168,16 +168,21 @@ const statements = {
"importData": `
REPLACE INTO collection_data (collection_name, record_id, record)
VALUES (:collection_name, :record_id, :record);`,
"scanAllRecords": `SELECT * FROM collection_data;`,
"clearCollectionMetadata": `DELETE FROM collection_metadata;`,
+
+ "calculateStorage": `
+ SELECT collection_name, SUM(LENGTH(record)) as size, COUNT(record) as num_records
+ FROM collection_data
+ GROUP BY collection_name;`,
};
const createStatements = [
"createCollectionData",
"createCollectionMetadata",
"createCollectionDataRecordIdIndex",
];
@@ -375,16 +380,27 @@ class FirefoxAdapter extends Kinto.adapt
.then(result => {
if (result.length == 0) {
return 0;
}
return result[0].getResultByName("last_modified");
});
}
+ calculateStorage() {
+ return this._executeStatement(statements.calculateStorage, {})
+ .then(result => {
+ return Array.from(result, row => ({
+ collectionName: row.getResultByName("collection_name"),
+ size: row.getResultByName("size"),
+ numRecords: row.getResultByName("num_records"),
+ }));
+ });
+ }
+
/**
* Reset the sync status of every record and collection we have
* access to.
*/
resetSyncStatus() {
// We're going to use execute instead of executeCached, so build
// in our own sanity check
if (!this._connection) {
--- a/toolkit/components/extensions/ExtensionStorageSync.jsm
+++ b/toolkit/components/extensions/ExtensionStorageSync.jsm
@@ -25,16 +25,22 @@ const STORAGE_SYNC_ENABLED_PREF = "webex
const STORAGE_SYNC_SERVER_URL_PREF = "webextensions.storage.sync.serverURL";
const STORAGE_SYNC_SCOPE = "sync:addon_storage";
const STORAGE_SYNC_CRYPTO_COLLECTION_NAME = "storage-sync-crypto";
const STORAGE_SYNC_CRYPTO_KEYRING_RECORD_ID = "keys";
const STORAGE_SYNC_CRYPTO_SALT_LENGTH_BYTES = 32;
const FXA_OAUTH_OPTIONS = {
scope: STORAGE_SYNC_SCOPE,
};
+const HISTOGRAM_GET_OPS_SIZE = "STORAGE_SYNC_GET_OPS_SIZE";
+const HISTOGRAM_SET_OPS_SIZE = "STORAGE_SYNC_SET_OPS_SIZE";
+const HISTOGRAM_REMOVE_OPS = "STORAGE_SYNC_REMOVE_OPS";
+const SCALAR_EXTENSIONS_USING = "storage.sync.api.usage.extensions_using";
+const SCALAR_ITEMS_STORED = "storage.sync.api.usage.items_stored";
+const SCALAR_STORAGE_CONSUMED = "storage.sync.api.usage.storage_consumed";
// Default is 5sec, which seems a bit aggressive on the open internet
const KINTO_REQUEST_TIMEOUT = 30000;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/ExtensionUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
@@ -56,16 +62,18 @@ XPCOMUtils.defineLazyModuleGetter(this,
XPCOMUtils.defineLazyModuleGetter(this, "Kinto",
"resource://services-common/kinto-offline-client.js");
XPCOMUtils.defineLazyModuleGetter(this, "FirefoxAdapter",
"resource://services-common/kinto-storage-adapter.js");
XPCOMUtils.defineLazyModuleGetter(this, "Log",
"resource://gre/modules/Log.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Observers",
"resource://services-common/observers.js");
+XPCOMUtils.defineLazyModuleGetter(this, "Services",
+ "resource://gre/modules/Services.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Sqlite",
"resource://gre/modules/Sqlite.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Svc",
"resource://services-sync/util.js");
XPCOMUtils.defineLazyModuleGetter(this, "Task",
"resource://gre/modules/Task.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Utils",
"resource://services-sync/util.js");
@@ -713,19 +721,22 @@ const openCollection = Task.async(functi
});
return coll;
});
class ExtensionStorageSync {
/**
* @param {FXAccounts} fxaService (Optional) If not
* present, trying to sync will fail.
+ * @param {nsITelemetry} telemetry Telemetry service to use to
+ * report sync usage.
*/
- constructor(fxaService) {
+ constructor(fxaService, telemetry) {
this._fxaService = fxaService;
+ this._telemetry = telemetry;
this.cryptoCollection = new CryptoCollection(fxaService);
this.listeners = new WeakMap();
}
async syncAll() {
const extensions = extensionContexts.keys();
const extIds = Array.from(extensions, extension => extension.id);
log.debug(`Syncing extension settings for ${JSON.stringify(extIds)}`);
@@ -736,16 +747,25 @@ class ExtensionStorageSync {
await this.ensureCanSync(extIds);
await this.checkSyncKeyRing();
const promises = Array.from(extensionContexts.keys(), extension => {
return openCollection(this.cryptoCollection, extension).then(coll => {
return this.sync(extension, coll);
});
});
await Promise.all(promises);
+
+ // This needs access to an adapter, but any adapter will do
+ const collection = await this.cryptoCollection.getCollection();
+ const storage = await collection.db.calculateStorage();
+ this._telemetry.scalarSet(SCALAR_EXTENSIONS_USING, storage.length);
+ for (let {collectionName, size, numRecords} of storage) {
+ this._telemetry.keyedScalarSet(SCALAR_ITEMS_STORED, collectionName, numRecords);
+ this._telemetry.keyedScalarSet(SCALAR_STORAGE_CONSUMED, collectionName, size);
+ }
}
async sync(extension, collection) {
throwIfNoFxA(this._fxaService, "syncing chrome.storage.sync");
const signedInUser = await this._fxaService.getSignedInUser();
if (!signedInUser) {
// FIXME: this should support syncing to self-hosted
log.info("User was not signed into FxA; cannot sync");
@@ -1072,21 +1092,23 @@ class ExtensionStorageSync {
return openCollection(this.cryptoCollection, extension, context);
}
async set(extension, items, context) {
const coll = await this.getCollection(extension, context);
const keys = Object.keys(items);
const ids = keys.map(keyToId);
+ const histogramSize = this._telemetry.getKeyedHistogramById(HISTOGRAM_SET_OPS_SIZE);
const changes = await coll.execute(txn => {
let changes = {};
for (let [i, key] of keys.entries()) {
const id = ids[i];
let item = items[key];
+ histogramSize.add(extension.id, JSON.stringify(item).length);
let {oldRecord} = txn.upsert({
id,
key,
data: item,
});
changes[key] = {
newValue: item,
};
@@ -1116,36 +1138,40 @@ class ExtensionStorageSync {
};
}
}
return changes;
}, {preloadIds: ids});
if (Object.keys(changes).length > 0) {
this.notifyListeners(extension, changes);
}
+ const histogram = this._telemetry.getKeyedHistogramById(HISTOGRAM_REMOVE_OPS);
+ histogram.add(extension.id, keys.length);
}
async clear(extension, context) {
// We can't call Collection#clear here, because that just clears
// the local database. We have to explicitly delete everything so
// that the deletions can be synced as well.
const coll = await this.getCollection(extension, context);
const res = await coll.list();
const records = res.data;
const keys = records.map(record => record.key);
await this.remove(extension, keys, context);
}
async get(extension, spec, context) {
const coll = await this.getCollection(extension, context);
+ const histogramSize = this._telemetry.getKeyedHistogramById(HISTOGRAM_GET_OPS_SIZE);
let keys, records;
if (spec === null) {
records = {};
const res = await coll.list();
for (let record of res.data) {
+ histogramSize.add(extension.id, JSON.stringify(record.data).length);
records[record.key] = record.data;
}
return records;
}
if (typeof spec === "string") {
keys = [spec];
records = {};
} else if (Array.isArray(spec)) {
@@ -1154,16 +1180,17 @@ class ExtensionStorageSync {
} else {
keys = Object.keys(spec);
records = Cu.cloneInto(spec, global);
}
for (let key of keys) {
const res = await coll.getAny(keyToId(key));
if (res.data && res.data._status != "deleted") {
+ histogramSize.add(extension.id, JSON.stringify(res.data.data).length);
records[res.data.key] = res.data.data;
}
}
return records;
}
addOnChangedListener(extension, listener, context) {
@@ -1189,9 +1216,9 @@ class ExtensionStorageSync {
if (listeners) {
for (let listener of listeners) {
runSafeSyncWithoutClone(listener, changes);
}
}
}
}
this.ExtensionStorageSync = ExtensionStorageSync;
-this.extensionStorageSync = new ExtensionStorageSync(_fxaService);
+this.extensionStorageSync = new ExtensionStorageSync(_fxaService, Services.telemetry);
--- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
@@ -386,17 +386,39 @@ function* withSignedInUser(user, f) {
},
getOAuthToken() {
return Promise.resolve("some-access-token");
},
sessionStatus() {
return Promise.resolve(true);
},
};
- let extensionStorageSync = new ExtensionStorageSync(fxaServiceMock);
+
+ let telemetryMock = {
+ _calls: [],
+ _histograms: {},
+ scalarSet(name, value) {
+ this._calls.push({method: "scalarSet", name, value});
+ },
+ keyedScalarSet(name, key, value) {
+ this._calls.push({method: "keyedScalarSet", name, key, value});
+ },
+ getKeyedHistogramById(name) {
+ let self = this;
+ return {
+ add(key, value) {
+ if (!self._histograms[name]) {
+ self._histograms[name] = [];
+ }
+ self._histograms[name].push(value);
+ },
+ };
+ },
+ };
+ let extensionStorageSync = new ExtensionStorageSync(fxaServiceMock, telemetryMock);
yield* f(extensionStorageSync, fxaServiceMock);
}
// Some assertions that make it easier to write tests about what was
// posted and when.
// Assert that the request was made with the correct access token.
// This should be true of all requests, so this is usually called from
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -10695,16 +10695,47 @@
},
"TOTAL_CONTAINERS_OPENED": {
"alert_emails": ["amarchesini@mozilla.com"],
"expires_in_version": "never",
"bug_numbers": [1276006],
"kind": "count",
"description": "Tracking the total number of opened Containers."
},
+ "STORAGE_SYNC_GET_OPS_SIZE": {
+ "alert_emails": ["eglassercamp@mozilla.com"],
+ "expires_in_version": "58",
+ "bug_numbers": [1328974],
+ "kind": "exponential",
+ "high": 100000,
+ "n_buckets": 30,
+ "keyed": true,
+ "description": "Track the size of results of get() operations performed this subsession. The key is the addon ID.",
+ "releaseChannelCollection": "opt-out"
+ },
+ "STORAGE_SYNC_SET_OPS_SIZE": {
+ "alert_emails": ["eglassercamp@mozilla.com"],
+ "expires_in_version": "58",
+ "bug_numbers": [1328974],
+ "kind": "exponential",
+ "high": 100000,
+ "n_buckets": 30,
+ "keyed": true,
+ "description": "Track the size of set() operations performed by addons this subsession. The key is the addon ID.",
+ "releaseChannelCollection": "opt-out"
+ },
+ "STORAGE_SYNC_REMOVE_OPS": {
+ "alert_emails": ["eglassercamp@mozilla.com"],
+ "expires_in_version": "58",
+ "bug_numbers": [1328974],
+ "kind": "count",
+ "keyed": true,
+ "description": "Track the number of remove() operations addons perform this subsession. The key is the addon ID.",
+ "releaseChannelCollection": "opt-out"
+ },
"FENNEC_SESSIONSTORE_DAMAGED_SESSION_FILE": {
"alert_emails": ["jh+bugzilla@buttercookie.de"],
"expires_in_version": "56",
"kind": "flag",
"bug_numbers": [1284017],
"description": "When restoring tabs on startup, reading from sessionstore.js failed, even though the file exists and is not containing an explicitly empty window.",
"cpp_guard": "ANDROID"
},
--- a/toolkit/components/telemetry/Scalars.yaml
+++ b/toolkit/components/telemetry/Scalars.yaml
@@ -196,16 +196,68 @@ browser.engagement.navigation:
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
record_in_processes:
- 'main'
+# This section is for probes used to measure use of the Webextensions storage.sync API.
+storage.sync.api.usage:
+ extensions_using:
+ bug_numbers:
+ - 1328974
+ description: >
+ The count of webextensions that have data stored in the chrome.storage.sync API.
+ This includes extensions that have not used the storage.sync API this session.
+ This includes items that were not stored this session.
+ This scalar is collected after every sync.
+ expires: "58"
+ kind: uint
+ keyed: false
+ notification_emails:
+ - eglassercamp@mozilla.com
+ release_channel_collection: opt-out
+ record_in_processes:
+ - main
+ items_stored:
+ bug_numbers:
+ - 1328974
+ description: >
+ The count of items in storage.sync storage, broken down by extension ID.
+ This includes extensions that have not used the storage.sync API this session.
+ This includes items that were not stored this session.
+ This scalar is collected after every sync.
+ expires: "58"
+ kind: uint
+ keyed: true
+ notification_emails:
+ - eglassercamp@mozilla.com
+ release_channel_collection: opt-out
+ record_in_processes:
+ - main
+ storage_consumed:
+ bug_numbers:
+ - 1328974
+ description: >
+ The count of bytes used in storage.sync, broken down by extension ID.
+ This includes extensions that have not used the storage.sync API this session.
+ This includes items that were not stored this session.
+ This scalar is collected after every sync.
+ expires: "58"
+ kind: uint
+ keyed: true
+ notification_emails:
+ - eglassercamp@mozilla.com
+ release_channel_collection: opt-out
+ record_in_processes:
+ - main
+
+
# The following section is for probes testing the Telemetry system. They will not be
# submitted in pings and are only used for testing.
telemetry.test:
unsigned_int_kind:
bug_numbers:
- 1276190
description: >
This is a test uint type with a really long description, maybe spanning even multiple