Bug 1453692 - Add RemoteSettings inspect() method r?Gijs draft
authorMathieu Leplatre <mathieu@mozilla.com>
Fri, 08 Jun 2018 17:08:33 +0200
changeset 808162 51505b8e4657089fb210db26d3e71a5ea027b35c
parent 808097 f804cc575bba9c6dcb4e3770f7dafe4d8934e73c
push id113296
push usermleplatre@mozilla.com
push dateMon, 18 Jun 2018 15:48:21 +0000
reviewersGijs
bugs1453692
milestone62.0a1
Bug 1453692 - Add RemoteSettings inspect() method r?Gijs MozReview-Commit-ID: FRHvXnGzBq0
services/common/tests/unit/test_blocklist_clients.js
services/settings/remote-settings.js
services/settings/test/unit/test_remote_settings.js
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -1,16 +1,17 @@
 const { Constructor: CC } = Components;
 
 ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 
+const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
 const BlocklistClients = ChromeUtils.import("resource://services-common/blocklist-clients.js", {});
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
 
 const IS_ANDROID = AppConstants.platform == "android";
 
 
@@ -80,20 +81,22 @@ function run_test() {
 
       response.write(responseBody);
       response.finish();
     } catch (e) {
       info(e);
     }
   }
   const configPath = "/v1/";
+  const monitorChangesPath = "/v1/buckets/monitor/collections/changes/records";
   const addonsRecordsPath  = "/v1/buckets/blocklists/collections/addons/records";
   const gfxRecordsPath     = "/v1/buckets/blocklists/collections/gfx/records";
   const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
   server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(monitorChangesPath, handleResponse);
   server.registerPathHandler(addonsRecordsPath, handleResponse);
   server.registerPathHandler(gfxRecordsPath, handleResponse);
   server.registerPathHandler(pluginsRecordsPath, handleResponse);
 
 
   run_next_test();
 
   registerCleanupFunction(function() {
@@ -285,16 +288,56 @@ add_task(async function test_entries_are
     await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
     const list = await client.get();
     equal(list.length, 4);
     ok(list.every(e => e.willMatch));
   }
 });
 add_task(clear_state);
 
+add_task(async function test_inspect_with_blocklist_clients() {
+  const rsSigner = "remote-settings.content-signature.mozilla.org";
+  const {
+    serverTimestamp,
+    localTimestamp,
+    lastCheck,
+    collections
+  } = await RemoteSettings.inspect();
+
+  equal(serverTimestamp, '"3000"');
+  equal(localTimestamp, null); // not yet synchronized.
+  equal(lastCheck, 0); // not yet synchronized.
+
+  const defaults = {
+    bucket: "blocklists",
+    localTimestamp: null,
+    lastCheck: 0,
+    signerName: rsSigner,
+  };
+  equal(collections.length, 3);
+  deepEqual(collections[0], { ...defaults, collection: "gfx", serverTimestamp: 3000 });
+  deepEqual(collections[1], { ...defaults, collection: "addons", serverTimestamp: 2900 });
+  deepEqual(collections[2], { ...defaults, collection: "plugins", serverTimestamp: 2800 });
+
+  // Now synchronize, and check that values were updated.
+  Services.prefs.setBoolPref("services.settings.load_dump", false);
+  const currentTime = Math.floor(Date.now() / 1000);
+  await RemoteSettings.pollChanges();
+
+  const inspected = await RemoteSettings.inspect();
+
+  equal(inspected.localTimestamp, '"3000"');
+  ok(inspected.lastCheck >= currentTime, `last check ${inspected.lastCheck}`);
+  for (const c of inspected.collections) {
+    equal(c.localTimestamp, 3000);
+    ok(c.lastCheck >= currentTime, `${c.collectionName} last check ${c.lastCheck}`);
+  }
+});
+add_task(clear_state);
+
 
 // get a response for a given request from sample data
 function getSampleResponse(req, port) {
   const responses = {
     "OPTIONS": {
       "sampleHeaders": [
         "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
         "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
@@ -319,16 +362,55 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto"
       })
     },
+    "GET:/v1/buckets/monitor/collections/changes/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        `Date: ${new Date().toUTCString()}`,
+        "Etag: \"3000\""
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": [{
+          "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+          "bucket": "blocklists",
+          "collection": "gfx",
+          "last_modified": 3000
+        }, {
+          "id": "36b2ebab-d691-4796-b36b-f7a06df38b26",
+          "bucket": "blocklists",
+          "collection": "addons",
+          "last_modified": 2900
+        }, {
+          "id": "42aea14b-4b35-4308-94d9-8562412a2fef",
+          "bucket": "blocklists",
+          "collection": "plugins",
+          "last_modified": 2800
+        }, {
+          "id": "50f4ef31-7788-4be8-b073-114440be4d8d",
+          "bucket": "main",
+          "collection": "passwords",
+          "last_modified": 2700
+        }, {
+          "id": "d2f08123-b815-4bbf-a0b7-a948a65ecafa",
+          "bucket": "pinning-preview",
+          "collection": "pins",
+          "last_modified": 2600
+        }]
+      })
+    },
     "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\""
       ],
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -596,61 +596,111 @@ async function hasLocalDump(bucket, coll
 
 function remoteSettingsFunction() {
   const _clients = new Map();
 
   // If not explicitly specified, use the default bucket name and signer.
   const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
   const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
 
+  /**
+   * RemoteSettings constructor.
+   *
+   * @param {String} collectionName The remote settings identifier
+   * @param {Object} options Advanced options
+   * @returns {RemoteSettingsClient} An instance of a Remote Settings client.
+   */
   const remoteSettings = function(collectionName, options) {
     // Get or instantiate a remote settings client.
     const rsOptions = {
       bucketName: mainBucket,
       signerName: defaultSigner,
       ...options
     };
     const { bucketName } = rsOptions;
     const key = `${bucketName}/${collectionName}`;
     if (!_clients.has(key)) {
       const c = new RemoteSettingsClient(collectionName, rsOptions);
       _clients.set(key, c);
     }
     return _clients.get(key);
   };
 
-  // This is called by the ping mechanism.
-  // returns a promise that rejects if something goes wrong
+  Object.defineProperty(remoteSettings, "pollingEndpoint", {
+    get() {
+      const kintoServer = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+      const changesPath = Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
+      return kintoServer + changesPath;
+    }
+  });
+
+  /**
+   * Internal helper to retrieve existing instances of clients or new instances
+   * with default options if possible, or `null` if bucket/collection are unknown.
+   */
+  async function _client(bucketName, collectionName) {
+    // Check if a client was registered for this bucket/collection. Potentially
+    // with some specific options like signer, filter function etc.
+    const key = `${bucketName}/${collectionName}`;
+    const client = _clients.get(key);
+    if (client) {
+      // If the bucket name was changed manually on the client instance and does not
+      // match, don't return it.
+      if (client.bucketName == bucketName) {
+        return client;
+      }
+
+    // There was no client registered for this bucket/collection, but it's the main bucket,
+    // therefore we can instantiate a client with the default options.
+    // So if we have a local database or if we ship a JSON dump, then it means that
+    // this client is known but it was not registered yet (eg. calling module not "imported" yet).
+    } else if (bucketName == mainBucket) {
+      const [dbExists, localDump] = await Promise.all([
+        databaseExists(bucketName, collectionName),
+        hasLocalDump(bucketName, collectionName)
+      ]);
+      if (dbExists || localDump) {
+        return new RemoteSettingsClient(collectionName, { bucketName, signerName: defaultSigner });
+      }
+    }
+    // Else, we cannot return a client insttance because we are not able to synchronize data in specific buckets.
+    // Mainly because we cannot guess which `signerName` has to be used for example.
+    // And we don't want to synchronize data for collections in the main bucket that are
+    // completely unknown (ie. no database and no JSON dump).
+    return null;
+  }
+
+  /**
+   * Main polling method, called by the ping mechanism.
+   *
+   * @returns {Promise} or throws error if something goes wrong.
+   */
   remoteSettings.pollChanges = async () => {
     // Check if the server backoff time is elapsed.
     if (Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
       const backoffReleaseTime = Services.prefs.getCharPref(PREF_SETTINGS_SERVER_BACKOFF);
       const remainingMilliseconds = parseInt(backoffReleaseTime, 10) - Date.now();
       if (remainingMilliseconds > 0) {
         // Backoff time has not elapsed yet.
         UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY,
                                UptakeTelemetry.STATUS.BACKOFF);
         throw new Error(`Server is asking clients to back off; retry in ${Math.ceil(remainingMilliseconds / 1000)}s.`);
       } else {
         Services.prefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
       }
     }
 
-    // Right now, we only use the collection name and the last modified info
-    const kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
-    const changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
-
     let lastEtag;
     if (Services.prefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
       lastEtag = Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
     }
 
     let pollResult;
     try {
-      pollResult = await fetchLatestChanges(changesEndpoint, lastEtag);
+      pollResult = await fetchLatestChanges(remoteSettings.pollingEndpoint, lastEtag);
     } catch (e) {
       // Report polling error to Uptake Telemetry.
       let report;
       if (/Server/.test(e.message)) {
         report = UptakeTelemetry.STATUS.SERVER_ERROR;
       } else if (/NetworkError/.test(e.message)) {
         report = UptakeTelemetry.STATUS.NETWORK_ERROR;
       } else {
@@ -677,54 +727,31 @@ function remoteSettingsFunction() {
     // Record new update time and the difference between local and server time.
     // Negative clockDifference means local time is behind server time
     // by the absolute of that value in seconds (positive means it's ahead)
     const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
     Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
     Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
 
     const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
+
     // Iterate through the collections version info and initiate a synchronization
     // on the related remote settings client.
     let firstError;
     for (const change of changes) {
-      const {bucket: bucketName, collection, last_modified: lastModified} = change;
-      const key = `${bucketName}/${collection}`;
-
-      let client;
-      // Check if a client was registered for this bucket/collection. Potentially
-      // with some specific options like bucket, signer, etc.
-      if (_clients.has(key)) {
-        client = _clients.get(key);
-        // If the bucket name was changed manually on the client instance and does not
-        // match, it should be skipped.
-        if (client.bucketName != bucketName) {
-          continue;
-        }
+      const { bucket, collection, last_modified } = change;
 
-      // There was no client registered for this bucket/collection, but it's the main bucket,
-      // therefore we can instantiate a client with the default options.
-      // So if we have a local database or if we ship a JSON dump, then it means that
-      // this client is known but it was not registered yet (eg. calling module not "imported" yet).
-      } else if (bucketName == mainBucket && (await databaseExists(bucketName, collection) ||
-                                              await hasLocalDump(bucketName, collection))) {
-        client = new RemoteSettingsClient(collection, {bucketName, signerName: defaultSigner});
-
-      // We are not able to synchronize data for clients in specific buckets since we cannot
-      // guess which `signerName` has to be used for example.
-      // And we don't want to synchronize data for collections in the main bucket that are
-      // completely unknown (ie. no database and no JSON dump).
-      } else {
+      const client = await _client(bucket, collection);
+      if (!client) {
         continue;
       }
-
       // Start synchronization! It will be a no-op if the specified `lastModified` equals
       // the one in the local database.
       try {
-        await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
+        await client.maybeSync(last_modified, serverTimeMillis, {loadDump});
       } catch (e) {
         if (!firstError) {
           firstError = e;
           firstError.details = change;
         }
       }
     }
     if (firstError) {
@@ -735,30 +762,66 @@ function remoteSettingsFunction() {
     // Save current Etag for next poll.
     if (currentEtag) {
       Services.prefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
     }
 
     Services.obs.notifyObservers(null, "remote-settings-changes-polled");
   };
 
+  /**
+   * Returns an object with polling status information and the list of
+   * known remote settings collections.
+   */
+  remoteSettings.inspect = async () => {
+    const { changes, currentEtag: serverTimestamp } = await fetchLatestChanges(remoteSettings.pollingEndpoint);
+
+    const collections = await Promise.all(changes.map(async (change) => {
+      const { bucket, collection, last_modified: serverTimestamp } = change;
+      const client = await _client(bucket, collection);
+      if (!client) {
+        return null;
+      }
+      const kintoCol = await client.openCollection();
+      const localTimestamp = await kintoCol.db.getLastModified();
+      const lastCheck = Services.prefs.getIntPref(client.lastCheckTimePref, 0);
+      return {
+        bucket,
+        collection,
+        localTimestamp,
+        serverTimestamp,
+        lastCheck,
+        signerName: client.signerName
+      };
+    }));
+
+    return {
+      serverURL: Services.prefs.getCharPref(PREF_SETTINGS_SERVER),
+      serverTimestamp,
+      localTimestamp: Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG, null),
+      lastCheck: Services.prefs.getIntPref(PREF_SETTINGS_LAST_UPDATE, 0),
+      mainBucket,
+      defaultSigner,
+      collections: collections.filter(c => !!c)
+    };
+  };
+
 
   const broadcastID = "remote-settings/monitor_changes";
   // When we start on a new profile there will be no ETag stored.
   // Use an arbitrary ETag that is guaranteed not to occur.
   // This will trigger a broadcast message but that's fine because we
   // will check the changes on each collection and retrieve only the
   // changes (e.g. nothing if we have a dump with the same data).
   const currentVersion = Services.prefs.getStringPref(PREF_SETTINGS_LAST_ETAG, "\"0\"");
   const moduleInfo = {
     moduleURI: __URI__,
     symbolName: "remoteSettingsBroadcastHandler",
   };
-  pushBroadcastService.addListener(broadcastID, currentVersion,
-                                   moduleInfo);
+  pushBroadcastService.addListener(broadcastID, currentVersion, moduleInfo);
 
   return remoteSettings;
 }
 
 var RemoteSettings = remoteSettingsFunction();
 
 var remoteSettingsBroadcastHandler = {
   async receivedBroadcastMessage(data, broadcastID) {
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -56,18 +56,20 @@ function run_test() {
 
       response.write(JSON.stringify(sample.responseBody));
       response.finish();
     } catch (e) {
       info(e);
     }
   }
   const configPath = "/v1/";
+  const changesPath = "/v1/buckets/monitor/collections/changes/records";
   const recordsPath  = "/v1/buckets/main/collections/password-fields/records";
   server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(changesPath, handleResponse);
   server.registerPathHandler(recordsPath, handleResponse);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
@@ -140,16 +142,39 @@ add_task(async function test_sync_event_
   equal(eventData.current.length, 1);
   equal(eventData.created.length, 0);
   equal(eventData.updated.length, 0);
   equal(eventData.deleted.length, 1);
   equal(eventData.deleted[0].website, "https://www.other.org/signin");
 });
 add_task(clear_state);
 
+add_task(async function test_inspect_method() {
+  const serverTime = Date.now();
+
+  // Synchronize the `password-fields` collection.
+  await client.maybeSync(Infinity, serverTime);
+
+  const inspected = await RemoteSettings.inspect();
+
+  const { mainBucket, serverURL, defaultSigner, collections } = inspected;
+  const rsSigner = "remote-settings.content-signature.mozilla.org";
+  equal(mainBucket, "main");
+  equal(serverURL, `http://localhost:${server.identity.primaryPort}/v1`);
+  equal(defaultSigner, rsSigner);
+
+  equal(inspected.serverTimestamp, '"4000"');
+  equal(collections.length, 1);
+  // password-fields was synchronized and has local data.
+  equal(collections[0].collection, "password-fields");
+  equal(collections[0].serverTimestamp, 3000);
+  equal(collections[0].localTimestamp, 3000);
+});
+add_task(clear_state);
+
 add_task(async function test_all_listeners_are_executed_if_one_fails() {
   const serverTime = Date.now();
 
   let count = 0;
   client.on("sync", () => { count += 1; });
   client.on("sync", () => { throw new Error("boom"); });
   client.on("sync", () => { count += 2; });
 
@@ -270,16 +295,40 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto"
       }
     },
+    "GET:/v1/buckets/monitor/collections/changes/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        `Date: ${new Date().toUTCString()}`,
+        "Etag: \"4000\""
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": {
+        "data": [{
+          "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+          "bucket": "main",
+          "collection": "unknown",
+          "last_modified": 4000
+        }, {
+          "id": "0af8da0b-3e03-48fb-8d0d-2d8e4cb7514d",
+          "bucket": "main",
+          "collection": "password-fields",
+          "last_modified": 3000
+        }]
+      }
+    },
     "GET:/v1/buckets/main/collections/password-fields/records?_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\""
       ],