Bug 1450998: Part 2: Introduce RemoteSettings("key").get()
MozReview-Commit-ID: 1V19BZVtBU4
new file mode 100644
--- /dev/null
+++ b/services/common/docs/RemoteSettings.rst
@@ -0,0 +1,111 @@
+.. _services/remotesettings:
+
+===============
+Remote Settings
+===============
+
+The `remote-settings.js <https://dxr.mozilla.org/mozilla-central/source/services/common/remote-settings.js>`_ module offers the ability to fetch remote settings that are kept in sync with Mozilla servers.
+
+
+Usage
+=====
+
+The `get()` method returns the list of entries for a specific key. Each entry can have arbitrary attributes, and can only be modified on the server.
+
+.. code-block:: js
+
+ const { RemoteSettings } = ChromeUtils.import("resource://services-common/remote-settings.js", {});
+
+ const data = await RemoteSettings("a-key").get();
+
+ /*
+ data == [
+ {label: "Yahoo", enabled: true, weight: 10, id: "d0782d8d", last_modified: 1522764475905},
+ {label: "Google", enabled: true, weight: 20, id: "8883955f", last_modified: 1521539068414},
+ {label: "Ecosia", enabled: false, weight: 5, id: "337c865d", last_modified: 1520527480321},
+ ]
+ */
+
+ for(const entry of data) {
+ // Do something with entry...
+ // await InternalAPI.load(entry.id, entry.label, entry.weight);
+ });
+
+.. note::
+ The ``id`` and ``last_modified`` (timestamp) attributes are assigned by the server.
+
+Options
+-------
+
+The list can optionally be filtered or ordered:
+
+.. code-block:: js
+
+ const subset = await RemoteSettings("a-key").get({
+ filters: {
+ "enabled": true,
+ },
+ order: "-weight"
+ });
+
+Events
+------
+
+The ``change`` event allows to be notified when the remote settings are changed. The event ``data`` attribute contains the whole new list of settings.
+
+.. code-block:: js
+
+ RemoteSettings("a-key").on("change", event => {
+ const { data } = event;
+ for(const entry of data) {
+ // Do something with entry...
+ // await InternalAPI.reload(entry.id, entry.label, entry.weight);
+ }
+ });
+
+.. note::
+ Currently, the update of remote settings is triggered by the `nsBlocklistService <https://dxr.mozilla.org/mozilla-central/source/toolkit/mozapps/extensions/nsBlocklistService.js>`_ (~ every 24H).
+
+File attachments
+----------------
+
+When an entry has a file attached to it, it has an ``attachment`` attribute, which contains the file related information (url, hash, size, mimetype, etc.). Remote files are not downloaded automatically.
+
+.. code-block:: js
+
+ const data = await RemoteSettings("a-key").get();
+
+ data.filter(d => d.attachment)
+ .forEach(async ({ attachment: { url, filename, size } }) => {
+ if (size < OS.freeDiskSpace) {
+ await downloadLocally(url, filename);
+ }
+ });
+
+
+Uptake Telemetry
+================
+
+Some :ref:`uptake telemetry <telemetry/collection/uptake>` is collected in order to monitor how remote settings are propagated.
+
+It is submitted to a single :ref:`keyed histogram <histogram-type-keyed>` whose id is ``UPTAKE_REMOTE_CONTENT_RESULT_1`` and the keys are prefixed with ``main/`` (eg. ``main/a-key`` in the above example).
+
+
+Create new remote settings
+==========================
+
+Staff members can create new kinds of remote settings, following `this documentation <mana docs>`_.
+
+It basically consists in:
+
+#. Choosing a key (eg. ``search-providers``)
+#. Assigning collaborators to editors and reviewers groups
+#. (*optional*) Define a JSONSchema to validate entries
+#. (*optional*) Allow attachments on entries
+
+And once done:
+
+#. Create, modify or delete entries and let reviewers approve the changes
+#. Wait for Firefox to pick-up the changes for your settings key
+
+.. _mana docs: https://mana.mozilla.org/wiki/pages/viewpage.action?pageId=66655528
new file mode 100644
--- /dev/null
+++ b/services/common/docs/index.rst
@@ -0,0 +1,10 @@
+========
+Services
+========
+
+This is the nascent documentation of the Firefox services.
+
+.. toctree::
+ :maxdepth: 1
+
+ RemoteSettings
--- a/services/common/moz.build
+++ b/services/common/moz.build
@@ -37,8 +37,9 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'andr
TESTING_JS_MODULES.services.common += [
'modules-testing/logging.js',
]
JS_PREFERENCE_FILES += [
'services-common.js',
]
+SPHINX_TREES['services'] = 'docs'
--- a/services/common/remote-settings.js
+++ b/services/common/remote-settings.js
@@ -185,16 +185,34 @@ class RemoteSettingsClient {
} finally {
if (sqliteHandle) {
await sqliteHandle.close();
}
}
}
/**
+ * Lists settings.
+ *
+ * @param {Object} options The options object.
+ * @param {Object} options.filters Filter the results (default: `{}`).
+ * @param {Object} options.order The order to apply (default: `-last_modified`).
+ * @return {Promise}
+ */
+ async get(options = {}) {
+ // In Bug 1451031, we will do some jexl filtering to limit the list items
+ // whose target is matched.
+ const { filters, order } = options;
+ return this.openCollection(async c => {
+ const { data } = await c.list({ filters, order });
+ return data;
+ });
+ }
+
+ /**
* Synchronize from Kinto server, if necessary.
*
* @param {int} lastModified the lastModified date (on the server) for
the remote collection.
* @param {Date} serverTime the current date return by the server.
* @param {Object} options additional advanced options.
* @param {bool} options.loadDump load initial dump from disk on first sync (default: true)
* @return {Promise} which rejects on sync or process failure.
@@ -243,17 +261,17 @@ class RemoteSettingsClient {
reportStatus = UptakeTelemetry.STATUS.UP_TO_DATE;
return;
}
// Fetch changes from server.
try {
// Server changes have priority during synchronization.
const strategy = Kinto.syncStrategy.SERVER_WINS;
- const {ok} = await collection.sync({remote, strategy});
+ const { ok } = await collection.sync({remote, strategy});
if (!ok) {
// Some synchronization conflicts occured.
reportStatus = UptakeTelemetry.STATUS.CONFLICT_ERROR;
throw new Error("Sync failed");
}
} catch (e) {
if (e.message == INVALID_SIGNATURE) {
// Signature verification failed during synchronzation.
@@ -285,17 +303,17 @@ class RemoteSettingsClient {
reportStatus = UptakeTelemetry.STATUS.BACKOFF;
} else {
reportStatus = UptakeTelemetry.STATUS.SYNC_ERROR;
}
throw e;
}
}
// Read local collection of records.
- const {data} = await collection.list();
+ const { data } = await collection.list();
// Handle the obtained records (ie. apply locally).
try {
// Execute callbacks in order and sequentially.
// If one fails everything fails.
const callbacks = this._callbacks.get("change");
for (const cb of callbacks) {
await cb({ data });
--- a/services/common/tests/unit/test_blocklist_certificates.js
+++ b/services/common/tests/unit/test_blocklist_certificates.js
@@ -48,23 +48,21 @@ add_task(async function test_something()
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
// Test an empty db populates
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
- await OneCRLBlocklistClient.openCollection(async (collection) => {
- // Open the collection, verify it's been populated:
- const list = await collection.list();
- // We know there will be initial values from the JSON dump.
- // (at least as many as in the dump shipped when this test was written).
- Assert.ok(list.data.length >= 363);
- });
+ // Open the collection, verify it's been populated:
+ const list = await OneCRLBlocklistClient.get();
+ // We know there will be initial values from the JSON dump.
+ // (at least as many as in the dump shipped when this test was written).
+ Assert.ok(list.length >= 363);
// No sync will be intented if maybeSync() is up-to-date.
Services.prefs.clearUserPref("services.settings.server");
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
// Use any last_modified older than highest shipped in JSON dump.
await OneCRLBlocklistClient.maybeSync(123456, Date.now());
// Last check value was updated.
Assert.notEqual(0, Services.prefs.getIntPref("services.blocklist.onecrl.checked"));
@@ -78,32 +76,28 @@ add_task(async function test_something()
await collection.clear();
// a lastModified value of 1000 means we get a remote collection with a
// single record
await collection.db.saveLastModified(1000);
});
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
- await OneCRLBlocklistClient.openCollection(async (collection) => {
- // Open the collection, verify it's been updated:
- // Our test data now has two records; both should be in the local collection
- const list = await collection.list();
- Assert.equal(list.data.length, 1);
- });
+ // Open the collection, verify it's been updated:
+ // Our test data now has two records; both should be in the local collection
+ const before = await OneCRLBlocklistClient.get();
+ Assert.equal(before.length, 1);
// Test the db is updated when we call again with a later lastModified value
await OneCRLBlocklistClient.maybeSync(4000, Date.now());
- await OneCRLBlocklistClient.openCollection(async (collection) => {
- // Open the collection, verify it's been updated:
- // Our test data now has two records; both should be in the local collection
- const list = await collection.list();
- Assert.equal(list.data.length, 3);
- });
+ // Open the collection, verify it's been updated:
+ // Our test data now has two records; both should be in the local collection
+ const after = await OneCRLBlocklistClient.get();
+ Assert.equal(after.length, 3);
// Try to maybeSync with the current lastModified value - no connection
// should be attempted.
// Clear the kinto base pref so any connections will cause a test failure
Services.prefs.clearUserPref("services.settings.server");
await OneCRLBlocklistClient.maybeSync(4000, Date.now());
// Try again with a lastModified value at some point in the past
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -98,36 +98,32 @@ function run_test() {
});
}
add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is_empty() {
for (let {client} of gBlocklistClients) {
// Test an empty db populates, but don't reach server (specified timestamp <= dump).
await client.maybeSync(1, Date.now());
- // Open the collection, verify the loaded data has status to synced:
- await client.openCollection(async (collection) => {
- const list = await collection.list();
- equal(list.data[0]._status, "synced");
- });
+ // Verify the loaded data has status to synced:
+ const list = await client.get();
+ equal(list[0]._status, "synced");
}
});
add_task(clear_state);
add_task(async function test_records_obtained_from_server_are_stored_in_db() {
for (let {client} of gBlocklistClients) {
// Test an empty db populates
await client.maybeSync(2000, Date.now(), {loadDump: false});
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
- await client.openCollection(async (collection) => {
- const list = await collection.list();
- equal(list.data.length, 1);
- });
+ const list = await client.get();
+ equal(list.length, 1);
}
});
add_task(clear_state);
add_task(async function test_records_changes_are_overwritten_by_server_changes() {
const {client} = gBlocklistClients[0];
// Create some local conflicting data, and make sure it syncs without error.
--- a/services/common/tests/unit/test_blocklist_pinning.js
+++ b/services/common/tests/unit/test_blocklist_pinning.js
@@ -79,34 +79,30 @@ add_task(async function test_something()
ok(!sss.isSecureURI(sss.HEADER_HSTS,
Services.io.newURI("https://five.example.com"), 0));
// Test an empty db populates
await PinningPreloadClient.maybeSync(2000, Date.now());
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
- await PinningPreloadClient.openCollection(async (collection) => {
- const list = await collection.list();
- Assert.equal(list.data.length, 1);
- });
+ const before = await PinningPreloadClient.get();
+ Assert.equal(before.length, 1);
// check that a pin exists for one.example.com
ok(sss.isSecureURI(sss.HEADER_HPKP,
Services.io.newURI("https://one.example.com"), 0));
// Test the db is updated when we call again with a later lastModified value
await PinningPreloadClient.maybeSync(4000, Date.now());
// Open the collection, verify it's been updated:
// Our data now has four new records; all should be in the local collection
- await PinningPreloadClient.openCollection(async (collection) => {
- const list = await collection.list();
- Assert.equal(list.data.length, 5);
- });
+ const after = await PinningPreloadClient.get();
+ Assert.equal(after.length, 5);
// check that a pin exists for two.example.com and three.example.com
ok(sss.isSecureURI(sss.HEADER_HPKP,
Services.io.newURI("https://two.example.com"), 0));
ok(sss.isSecureURI(sss.HEADER_HPKP,
Services.io.newURI("https://three.example.com"), 0));
// check that a pin does not exist for four.example.com - it's in the
--- a/services/common/tests/unit/test_blocklist_signatures.js
+++ b/services/common/tests/unit/test_blocklist_signatures.js
@@ -46,21 +46,19 @@ function getCertChain() {
const chain = [];
for (let file of CHAIN_FILES) {
chain.push(getFileData(do_get_file(CERT_DIR + file)));
}
return chain.join("\n");
}
async function checkRecordCount(client, count) {
- await client.openCollection(async (collection) => {
- // Check we have the expected number of records
- const records = await collection.list();
- Assert.equal(count, records.data.length);
- });
+ // Check we have the expected number of records
+ const records = await client.get();
+ Assert.equal(count, records.length);
}
// Check to ensure maybeSync is called with correct values when a changes
// document contains information on when a collection was last modified
add_task(async function test_check_signatures() {
const port = server.identity.primaryPort;
const OneCRLBlocklistClient = BlocklistClients.OneCRLBlocklistClient;