--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -1,16 +1,17 @@
const { Constructor: CC } = Components;
ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm");
ChromeUtils.import("resource://testing-common/httpd.js");
const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
+const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
const BlocklistClients = ChromeUtils.import("resource://services-common/blocklist-clients.js", {});
const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream", "setInputStream");
const IS_ANDROID = AppConstants.platform == "android";
@@ -80,20 +81,22 @@ function run_test() {
response.write(responseBody);
response.finish();
} catch (e) {
info(e);
}
}
const configPath = "/v1/";
+ const monitorChangesPath = "/v1/buckets/monitor/collections/changes/records";
const addonsRecordsPath = "/v1/buckets/blocklists/collections/addons/records";
const gfxRecordsPath = "/v1/buckets/blocklists/collections/gfx/records";
const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
server.registerPathHandler(configPath, handleResponse);
+ server.registerPathHandler(monitorChangesPath, handleResponse);
server.registerPathHandler(addonsRecordsPath, handleResponse);
server.registerPathHandler(gfxRecordsPath, handleResponse);
server.registerPathHandler(pluginsRecordsPath, handleResponse);
run_next_test();
registerCleanupFunction(function() {
@@ -285,16 +288,56 @@ add_task(async function test_entries_are
await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
const list = await client.get();
equal(list.length, 4);
ok(list.every(e => e.willMatch));
}
});
add_task(clear_state);
+add_task(async function test_inspect_with_blocklist_clients() {
+ const rsSigner = "remote-settings.content-signature.mozilla.org";
+ const {
+ serverTimestamp,
+ localTimestamp,
+ lastCheck,
+ collections
+ } = await RemoteSettings.inspect();
+
+ equal(serverTimestamp, '"3000"');
+ equal(localTimestamp, null); // not yet synchronized.
+ equal(lastCheck, 0); // not yet synchronized.
+
+ const defaults = {
+ bucket: "blocklists",
+ localTimestamp: null,
+ lastCheck: 0,
+ signerName: rsSigner,
+ };
+ equal(collections.length, 3);
+ deepEqual(collections[0], { ...defaults, collection: "gfx", serverTimestamp: 3000 });
+ deepEqual(collections[1], { ...defaults, collection: "addons", serverTimestamp: 2900 });
+ deepEqual(collections[2], { ...defaults, collection: "plugins", serverTimestamp: 2800 });
+
+ // Now synchronize, and check that values were updated.
+ Services.prefs.setBoolPref("services.settings.load_dump", false);
+ const currentTime = Math.floor(Date.now() / 1000);
+ await RemoteSettings.pollChanges();
+
+ const inspected = await RemoteSettings.inspect();
+
+ equal(inspected.localTimestamp, '"3000"');
+ ok(inspected.lastCheck >= currentTime, `last check ${inspected.lastCheck}`);
+ for (const c of inspected.collections) {
+ equal(c.localTimestamp, 3000);
+ ok(c.lastCheck >= currentTime, `${c.collectionName} last check ${c.lastCheck}`);
+ }
+});
+add_task(clear_state);
+
// get a response for a given request from sample data
function getSampleResponse(req, port) {
const responses = {
"OPTIONS": {
"sampleHeaders": [
"Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
"Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
@@ -319,16 +362,55 @@ function getSampleResponse(req, port) {
},
"url": `http://localhost:${port}/v1/`,
"documentation": "https://kinto.readthedocs.org/",
"version": "1.5.1",
"commit": "cbc6f58",
"hello": "kinto"
})
},
+ "GET:/v1/buckets/monitor/collections/changes/records": {
+ "sampleHeaders": [
+ "Access-Control-Allow-Origin: *",
+ "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+ "Content-Type: application/json; charset=UTF-8",
+ "Server: waitress",
+ `Date: ${new Date().toUTCString()}`,
+ "Etag: \"3000\""
+ ],
+ "status": { status: 200, statusText: "OK" },
+ "responseBody": JSON.stringify({
+ "data": [{
+ "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+ "bucket": "blocklists",
+ "collection": "gfx",
+ "last_modified": 3000
+ }, {
+ "id": "36b2ebab-d691-4796-b36b-f7a06df38b26",
+ "bucket": "blocklists",
+ "collection": "addons",
+ "last_modified": 2900
+ }, {
+ "id": "42aea14b-4b35-4308-94d9-8562412a2fef",
+ "bucket": "blocklists",
+ "collection": "plugins",
+ "last_modified": 2800
+ }, {
+ "id": "50f4ef31-7788-4be8-b073-114440be4d8d",
+ "bucket": "main",
+ "collection": "passwords",
+ "last_modified": 2700
+ }, {
+ "id": "d2f08123-b815-4bbf-a0b7-a948a65ecafa",
+ "bucket": "pinning-preview",
+ "collection": "pins",
+ "last_modified": 2600
+ }]
+ })
+ },
"GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -596,61 +596,111 @@ async function hasLocalDump(bucket, coll
function remoteSettingsFunction() {
const _clients = new Map();
// If not explicitly specified, use the default bucket name and signer.
const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
+ /**
+ * RemoteSettings constructor.
+ *
+ * @param {String} collectionName The remote settings identifier
+ * @param {Object} options Advanced options
+ * @returns {RemoteSettingsClient} An instance of a Remote Settings client.
+ */
const remoteSettings = function(collectionName, options) {
// Get or instantiate a remote settings client.
const rsOptions = {
bucketName: mainBucket,
signerName: defaultSigner,
...options
};
const { bucketName } = rsOptions;
const key = `${bucketName}/${collectionName}`;
if (!_clients.has(key)) {
const c = new RemoteSettingsClient(collectionName, rsOptions);
_clients.set(key, c);
}
return _clients.get(key);
};
- // This is called by the ping mechanism.
- // returns a promise that rejects if something goes wrong
+ Object.defineProperty(remoteSettings, "pollingEndpoint", {
+ get() {
+ const kintoServer = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+ const changesPath = Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
+ return kintoServer + changesPath;
+ }
+ });
+
+ /**
+ * Internal helper to retrieve existing instances of clients or new instances
+ * with default options if possible, or `null` if bucket/collection are unknown.
+ */
+ async function _client(bucketName, collectionName) {
+ // Check if a client was registered for this bucket/collection. Potentially
+ // with some specific options like signer, filter function etc.
+ const key = `${bucketName}/${collectionName}`;
+ const client = _clients.get(key);
+ if (client) {
+ // If the bucket name was changed manually on the client instance and does not
+ // match, don't return it.
+ if (client.bucketName == bucketName) {
+ return client;
+ }
+
+ // There was no client registered for this bucket/collection, but it's the main bucket,
+ // therefore we can instantiate a client with the default options.
+ // So if we have a local database or if we ship a JSON dump, then it means that
+ // this client is known but it was not registered yet (eg. calling module not "imported" yet).
+ } else if (bucketName == mainBucket) {
+ const [dbExists, localDump] = await Promise.all([
+ databaseExists(bucketName, collectionName),
+ hasLocalDump(bucketName, collectionName)
+ ]);
+ if (dbExists || localDump) {
+ return new RemoteSettingsClient(collectionName, { bucketName, signerName: defaultSigner });
+ }
+ }
+ // Else, we cannot return a client insttance because we are not able to synchronize data in specific buckets.
+ // Mainly because we cannot guess which `signerName` has to be used for example.
+ // And we don't want to synchronize data for collections in the main bucket that are
+ // completely unknown (ie. no database and no JSON dump).
+ return null;
+ }
+
+ /**
+ * Main polling method, called by the ping mechanism.
+ *
+ * @returns {Promise} or throws error if something goes wrong.
+ */
remoteSettings.pollChanges = async () => {
// Check if the server backoff time is elapsed.
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
const backoffReleaseTime = Services.prefs.getCharPref(PREF_SETTINGS_SERVER_BACKOFF);
const remainingMilliseconds = parseInt(backoffReleaseTime, 10) - Date.now();
if (remainingMilliseconds > 0) {
// Backoff time has not elapsed yet.
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY,
UptakeTelemetry.STATUS.BACKOFF);
throw new Error(`Server is asking clients to back off; retry in ${Math.ceil(remainingMilliseconds / 1000)}s.`);
} else {
Services.prefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
}
}
- // Right now, we only use the collection name and the last modified info
- const kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
- const changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
-
let lastEtag;
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
lastEtag = Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
}
let pollResult;
try {
- pollResult = await fetchLatestChanges(changesEndpoint, lastEtag);
+ pollResult = await fetchLatestChanges(remoteSettings.pollingEndpoint, lastEtag);
} catch (e) {
// Report polling error to Uptake Telemetry.
let report;
if (/Server/.test(e.message)) {
report = UptakeTelemetry.STATUS.SERVER_ERROR;
} else if (/NetworkError/.test(e.message)) {
report = UptakeTelemetry.STATUS.NETWORK_ERROR;
} else {
@@ -677,54 +727,31 @@ function remoteSettingsFunction() {
// Record new update time and the difference between local and server time.
// Negative clockDifference means local time is behind server time
// by the absolute of that value in seconds (positive means it's ahead)
const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
+
// Iterate through the collections version info and initiate a synchronization
// on the related remote settings client.
let firstError;
for (const change of changes) {
- const {bucket: bucketName, collection, last_modified: lastModified} = change;
- const key = `${bucketName}/${collection}`;
-
- let client;
- // Check if a client was registered for this bucket/collection. Potentially
- // with some specific options like bucket, signer, etc.
- if (_clients.has(key)) {
- client = _clients.get(key);
- // If the bucket name was changed manually on the client instance and does not
- // match, it should be skipped.
- if (client.bucketName != bucketName) {
- continue;
- }
+ const { bucket, collection, last_modified } = change;
- // There was no client registered for this bucket/collection, but it's the main bucket,
- // therefore we can instantiate a client with the default options.
- // So if we have a local database or if we ship a JSON dump, then it means that
- // this client is known but it was not registered yet (eg. calling module not "imported" yet).
- } else if (bucketName == mainBucket && (await databaseExists(bucketName, collection) ||
- await hasLocalDump(bucketName, collection))) {
- client = new RemoteSettingsClient(collection, {bucketName, signerName: defaultSigner});
-
- // We are not able to synchronize data for clients in specific buckets since we cannot
- // guess which `signerName` has to be used for example.
- // And we don't want to synchronize data for collections in the main bucket that are
- // completely unknown (ie. no database and no JSON dump).
- } else {
+ const client = await _client(bucket, collection);
+ if (!client) {
continue;
}
-
// Start synchronization! It will be a no-op if the specified `lastModified` equals
// the one in the local database.
try {
- await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
+ await client.maybeSync(last_modified, serverTimeMillis, {loadDump});
} catch (e) {
if (!firstError) {
firstError = e;
firstError.details = change;
}
}
}
if (firstError) {
@@ -735,30 +762,66 @@ function remoteSettingsFunction() {
// Save current Etag for next poll.
if (currentEtag) {
Services.prefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
}
Services.obs.notifyObservers(null, "remote-settings-changes-polled");
};
+ /**
+ * Returns an object with polling status information and the list of
+ * known remote settings collections.
+ */
+ remoteSettings.inspect = async () => {
+ const { changes, currentEtag: serverTimestamp } = await fetchLatestChanges(remoteSettings.pollingEndpoint);
+
+ const collections = await Promise.all(changes.map(async (change) => {
+ const { bucket, collection, last_modified: serverTimestamp } = change;
+ const client = await _client(bucket, collection);
+ if (!client) {
+ return null;
+ }
+ const kintoCol = await client.openCollection();
+ const localTimestamp = await kintoCol.db.getLastModified();
+ const lastCheck = Services.prefs.getIntPref(client.lastCheckTimePref, 0);
+ return {
+ bucket,
+ collection,
+ localTimestamp,
+ serverTimestamp,
+ lastCheck,
+ signerName: client.signerName
+ };
+ }));
+
+ return {
+ serverURL: Services.prefs.getCharPref(PREF_SETTINGS_SERVER),
+ serverTimestamp,
+ localTimestamp: Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG, null),
+ lastCheck: Services.prefs.getIntPref(PREF_SETTINGS_LAST_UPDATE, 0),
+ mainBucket,
+ defaultSigner,
+ collections: collections.filter(c => !!c)
+ };
+ };
+
const broadcastID = "remote-settings/monitor_changes";
// When we start on a new profile there will be no ETag stored.
// Use an arbitrary ETag that is guaranteed not to occur.
// This will trigger a broadcast message but that's fine because we
// will check the changes on each collection and retrieve only the
// changes (e.g. nothing if we have a dump with the same data).
const currentVersion = Services.prefs.getStringPref(PREF_SETTINGS_LAST_ETAG, "\"0\"");
const moduleInfo = {
moduleURI: __URI__,
symbolName: "remoteSettingsBroadcastHandler",
};
- pushBroadcastService.addListener(broadcastID, currentVersion,
- moduleInfo);
+ pushBroadcastService.addListener(broadcastID, currentVersion, moduleInfo);
return remoteSettings;
}
var RemoteSettings = remoteSettingsFunction();
var remoteSettingsBroadcastHandler = {
async receivedBroadcastMessage(data, broadcastID) {
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -56,18 +56,20 @@ function run_test() {
response.write(JSON.stringify(sample.responseBody));
response.finish();
} catch (e) {
info(e);
}
}
const configPath = "/v1/";
+ const changesPath = "/v1/buckets/monitor/collections/changes/records";
const recordsPath = "/v1/buckets/main/collections/password-fields/records";
server.registerPathHandler(configPath, handleResponse);
+ server.registerPathHandler(changesPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
run_next_test();
registerCleanupFunction(function() {
server.stop(() => { });
});
}
@@ -140,16 +142,39 @@ add_task(async function test_sync_event_
equal(eventData.current.length, 1);
equal(eventData.created.length, 0);
equal(eventData.updated.length, 0);
equal(eventData.deleted.length, 1);
equal(eventData.deleted[0].website, "https://www.other.org/signin");
});
add_task(clear_state);
+add_task(async function test_inspect_method() {
+ const serverTime = Date.now();
+
+ // Synchronize the `password-fields` collection.
+ await client.maybeSync(Infinity, serverTime);
+
+ const inspected = await RemoteSettings.inspect();
+
+ const { mainBucket, serverURL, defaultSigner, collections } = inspected;
+ const rsSigner = "remote-settings.content-signature.mozilla.org";
+ equal(mainBucket, "main");
+ equal(serverURL, `http://localhost:${server.identity.primaryPort}/v1`);
+ equal(defaultSigner, rsSigner);
+
+ equal(inspected.serverTimestamp, '"4000"');
+ equal(collections.length, 1);
+ // password-fields was synchronized and has local data.
+ equal(collections[0].collection, "password-fields");
+ equal(collections[0].serverTimestamp, 3000);
+ equal(collections[0].localTimestamp, 3000);
+});
+add_task(clear_state);
+
add_task(async function test_all_listeners_are_executed_if_one_fails() {
const serverTime = Date.now();
let count = 0;
client.on("sync", () => { count += 1; });
client.on("sync", () => { throw new Error("boom"); });
client.on("sync", () => { count += 2; });
@@ -270,16 +295,40 @@ function getSampleResponse(req, port) {
},
"url": `http://localhost:${port}/v1/`,
"documentation": "https://kinto.readthedocs.org/",
"version": "1.5.1",
"commit": "cbc6f58",
"hello": "kinto"
}
},
+ "GET:/v1/buckets/monitor/collections/changes/records": {
+ "sampleHeaders": [
+ "Access-Control-Allow-Origin: *",
+ "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+ "Content-Type: application/json; charset=UTF-8",
+ "Server: waitress",
+ `Date: ${new Date().toUTCString()}`,
+ "Etag: \"4000\""
+ ],
+ "status": { status: 200, statusText: "OK" },
+ "responseBody": {
+ "data": [{
+ "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+ "bucket": "main",
+ "collection": "unknown",
+ "last_modified": 4000
+ }, {
+ "id": "0af8da0b-3e03-48fb-8d0d-2d8e4cb7514d",
+ "bucket": "main",
+ "collection": "password-fields",
+ "last_modified": 3000
+ }]
+ }
+ },
"GET:/v1/buckets/main/collections/password-fields/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],