Bug 1454970 - Decouple Remote Settings synchronization from initialization order r=glasserc,mgoodwin draft
authorMathieu Leplatre <mathieu@mozilla.com>
Thu, 24 May 2018 23:55:23 +0200
changeset 803977 f9a11de0976c872136cdd11261500cd2f601e0d9
parent 803743 d8f180ab74921fd07a66d6868914a48e5f9ea797
push id112252
push usermleplatre@mozilla.com
push dateTue, 05 Jun 2018 07:37:15 +0000
reviewersglasserc, mgoodwin
bugs1454970
milestone62.0a1
Bug 1454970 - Decouple Remote Settings synchronization from initialization order r=glasserc,mgoodwin MozReview-Commit-ID: LSwFflrFBMn
services/settings/remote-settings.js
services/settings/test/unit/test_remote_settings_poll.js
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -7,17 +7,17 @@
 var EXPORTED_SYMBOLS = [
   "RemoteSettings",
   "jexlFilterFunc"
 ];
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
-Cu.importGlobalProperties(["fetch"]);
+Cu.importGlobalProperties(["fetch", "indexedDB"]);
 
 ChromeUtils.defineModuleGetter(this, "Kinto",
                                "resource://services-common/kinto-offline-client.js");
 ChromeUtils.defineModuleGetter(this, "KintoHttpClient",
                                "resource://services-common/kinto-http-client.js");
 ChromeUtils.defineModuleGetter(this, "CanonicalJSON",
                                "resource://gre/modules/CanonicalJSON.jsm");
 ChromeUtils.defineModuleGetter(this, "UptakeTelemetry",
@@ -173,16 +173,31 @@ async function fetchLatestChanges(url, l
     if (!isNaN(value)) {
       backoffSeconds = value;
     }
   }
 
   return {changes, currentEtag, serverTimeMillis, backoffSeconds};
 }
 
+/**
+ * Load the the JSON file distributed with the release for this collection.
+ */
+async function loadDumpFile(filename) {
+  // Replace OS specific path separator by / for URI.
+  const { components: folderFile } = OS.Path.split(filename);
+  const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
+  const response = await fetch(fileURI);
+  if (!response.ok) {
+    throw new Error(`Could not read from '${fileURI}'`);
+  }
+  // Will be rejected if JSON is invalid.
+  return response.json();
+}
+
 
 class RemoteSettingsClient {
 
   constructor(collectionName, { bucketName, signerName, filterFunc = jexlFilterFunc, lastCheckTimePref }) {
     this.collectionName = collectionName;
     this.bucketName = bucketName;
     this.signerName = signerName;
     this.filterFunc = filterFunc;
@@ -270,18 +285,18 @@ class RemoteSettingsClient {
     const { filters = {}, order } = options;
     const c = await this.openCollection();
 
     const timestamp = await c.db.getLastModified();
     // If the local database was never synchronized, then we attempt to load
     // a packaged JSON dump.
     if (timestamp == null) {
       try {
-         const { data } = await this._loadDumpFile();
-         await c.loadDump(data);
+        const { data } = await loadDumpFile(this.filename);
+        await c.loadDump(data);
       } catch (e) {
         // Report but return an empty list since there will be no data anyway.
         Cu.reportError(e);
         return [];
       }
     }
 
     const { data } = await c.list({ filters, order });
@@ -321,17 +336,17 @@ class RemoteSettingsClient {
       let collectionLastModified = await collection.db.getLastModified();
 
       // If there is no data currently in the collection, attempt to import
       // initial data from the application defaults.
       // This allows to avoid synchronizing the whole collection content on
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          const initialData = await this._loadDumpFile();
+          const initialData = await loadDumpFile(this.filename);
           await collection.loadDump(initialData.data);
           collectionLastModified = await collection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
           Cu.reportError(e);
         }
       }
 
@@ -457,31 +472,16 @@ class RemoteSettingsClient {
       if (reportStatus === null) {
         reportStatus = UptakeTelemetry.STATUS.SUCCESS;
       }
       // Report success/error status to Telemetry.
       UptakeTelemetry.report(this.identifier, reportStatus);
     }
   }
 
-  /**
-   * Load the the JSON file distributed with the release for this collection.
-   */
-  async _loadDumpFile() {
-    // Replace OS specific path separator by / for URI.
-    const { components: folderFile } = OS.Path.split(this.filename);
-    const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
-    const response = await fetch(fileURI);
-    if (!response.ok) {
-      throw new Error(`Could not read from '${fileURI}'`);
-    }
-    // Will be rejected if JSON is invalid.
-    return response.json();
-  }
-
   async _validateCollectionSignature(remote, payload, collection, options = {}) {
     const {ignoreLocal} = options;
     // this is a content-signature field from an autograph response.
     const signaturePayload = await fetchCollectionMetadata(remote, collection);
     if (!signaturePayload) {
       throw new Error(MISSING_SIGNATURE);
     }
     const {x5u, signature} = signaturePayload;
@@ -534,16 +534,60 @@ class RemoteSettingsClient {
     }
     const environment = cacheProxy(ClientEnvironment);
     const dataPromises = data.map(e => this.filterFunc(e, environment));
     const results = await Promise.all(dataPromises);
     return results.filter(v => !!v);
   }
 }
 
+/**
+ * Check if an IndexedDB database exists for the specified bucket and collection.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @return {bool} Whether it exists or not.
+ */
+async function databaseExists(bucket, collection) {
+  // The dbname is chosen by kinto.js from the bucket and collection names.
+  // https://github.com/Kinto/kinto.js/blob/41aa1526e/src/collection.js#L231
+  const dbname = `${bucket}/${collection}`;
+  try {
+    await new Promise((resolve, reject) => {
+      const request = indexedDB.open(dbname, 1);
+      request.onupgradeneeded = event => {
+        event.target.transaction.abort();
+        reject(event.target.error);
+      };
+      request.onerror = event => reject(event.target.error);
+      request.onsuccess = event => resolve(event.target.result);
+    });
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Check if we ship a JSON dump for the specified bucket and collection.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @return {bool} Whether it is present or not.
+ */
+async function hasLocalDump(bucket, collection) {
+  const filename = OS.Path.join(bucket, `${collection}.json`);
+  try {
+    await loadDumpFile(filename);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
 
 function remoteSettingsFunction() {
   const _clients = new Map();
 
   // If not explicitly specified, use the default bucket name and signer.
   const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
   const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
 
@@ -627,30 +671,54 @@ function remoteSettingsFunction() {
     Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
     Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
 
     const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
     // Iterate through the collections version info and initiate a synchronization
     // on the related remote settings client.
     let firstError;
     for (const change of changes) {
-      const {bucket, collection, last_modified: lastModified} = change;
-      const key = `${bucket}/${collection}`;
-      if (!_clients.has(key)) {
+      const {bucket: bucketName, collection, last_modified: lastModified} = change;
+      const key = `${bucketName}/${collection}`;
+
+      let client;
+      // Check if a client was registered for this bucket/collection. Potentially
+      // with some specific options like bucket, signer, etc.
+      if (_clients.has(key)) {
+        client = _clients.get(key);
+        // If the bucket name was changed manually on the client instance and does not
+        // match, it should be skipped.
+        if (client.bucketName != bucketName) {
+          continue;
+        }
+
+      // There was no client registered for this bucket/collection, but it's the main bucket,
+      // therefore we can instantiate a client with the default options.
+      // So if we have a local database or if we ship a JSON dump, then it means that
+      // this client is known but it was not registered yet (eg. calling module not "imported" yet).
+      } else if (bucketName == mainBucket && (await databaseExists(bucketName, collection) ||
+                                              await hasLocalDump(bucketName, collection))) {
+        client = new RemoteSettingsClient(collection, {bucketName, signerName: defaultSigner});
+
+      // We are not able to synchronize data for clients in specific buckets since we cannot
+      // guess which `signerName` has to be used for example.
+      // And we don't want to synchronize data for collections in the main bucket that are
+      // completely unknown (ie. no database and no JSON dump).
+      } else {
         continue;
       }
-      const client = _clients.get(key);
-      if (client.bucketName != bucket) {
-        continue;
-      }
+
+      // Start synchronization! It will be a no-op if the specified `lastModified` equals
+      // the one in the local database.
       try {
         await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
       } catch (e) {
         if (!firstError) {
           firstError = e;
+          firstError.details = change;
         }
       }
     }
     if (firstError) {
       // cause the promise to reject by throwing the first observed error
       throw firstError;
     }
 
--- a/services/settings/test/unit/test_remote_settings_poll.js
+++ b/services/settings/test/unit/test_remote_settings_poll.js
@@ -1,239 +1,402 @@
 /* import-globals-from ../../../common/tests/unit/head_helpers.js */
 
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 
 const { UptakeTelemetry } = ChromeUtils.import("resource://services-common/uptake-telemetry.js", {});
 const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
+const { Kinto } = ChromeUtils.import("resource://services-common/kinto-offline-client.js", {});
 
-var server;
+const IS_ANDROID = AppConstants.platform == "android";
 
 const PREF_SETTINGS_SERVER = "services.settings.server";
 const PREF_SETTINGS_SERVER_BACKOFF = "services.settings.server.backoff";
 const PREF_LAST_UPDATE = "services.settings.last_update_seconds";
 const PREF_LAST_ETAG = "services.settings.last_etag";
 const PREF_CLOCK_SKEW_SECONDS = "services.settings.clock_skew_seconds";
 
 // Telemetry report result.
 const TELEMETRY_HISTOGRAM_KEY = "settings-changes-monitoring";
-
-// Check to ensure maybeSync is called with correct values when a changes
-// document contains information on when a collection was last modified
-add_task(async function test_check_maybeSync() {
-  const changesPath = "/v1/buckets/monitor/collections/changes/records";
-
-  // register a handler
-  function handleResponse(serverTimeMillis, request, response) {
-    try {
-      const sampled = getSampleResponse(request, server.identity.primaryPort);
-      if (!sampled) {
-        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
-      }
+const CHANGES_PATH = "/v1/buckets/monitor/collections/changes/records";
 
-      response.setStatusLine(null, sampled.status.status,
-                             sampled.status.statusText);
-      // send the headers
-      for (let headerLine of sampled.sampleHeaders) {
-        let headerElements = headerLine.split(":");
-        response.setHeader(headerElements[0], headerElements[1].trimLeft());
-      }
+var server;
 
-      // set the server date
-      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
-
-      response.write(sampled.responseBody);
-    } catch (e) {
-      dump(`${e}\n`);
-    }
-  }
-
-  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
-
+async function clear_state() {
   // set up prefs so the kinto updater talks to the test server
   Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
     `http://localhost:${server.identity.primaryPort}/v1`);
 
   // set some initial values so we can check these are updated appropriately
   Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
   Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0);
   Services.prefs.clearUserPref(PREF_LAST_ETAG);
-
-
-  let startTime = Date.now();
-
-  // ensure we get the maybeSync call
-  // add a test kinto client that will respond to lastModified information
-  // for a collection called 'test-collection'
-  const c = RemoteSettings("test-collection", {
-    bucketName: "test-bucket",
-  });
-  c.maybeSync = () => {};
-
-  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
-
-  let notificationObserved = false;
-
-  // Ensure that the remote-settings-changes-polled notification works
-  let certblockObserver = {
-    observe(aSubject, aTopic, aData) {
-      Services.obs.removeObserver(this, "remote-settings-changes-polled");
-      notificationObserved = true;
-    }
-  };
-
-  Services.obs.addObserver(certblockObserver, "remote-settings-changes-polled");
-
-  await RemoteSettings.pollChanges();
-
-  Assert.ok(notificationObserved, "a notification should have been observed");
-
-  // check the last_update is updated
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-  // How does the clock difference look?
-  let endTime = Date.now();
-  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
-  // we previously set the serverTime to 2 (seconds past epoch)
-  Assert.ok(clockDifference <= endTime / 1000
-              && clockDifference >= Math.floor(startTime / 1000) - 2);
-  // Last timestamp was saved. An ETag header value is a quoted string.
-  let lastEtag = Services.prefs.getCharPref(PREF_LAST_ETAG);
-  Assert.equal(lastEtag, "\"1100\"");
-
-  // Simulate a poll with up-to-date collection.
-  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
-  // If server has no change, a 304 is received, maybeSync() is not called.
-  c.maybeSync = () => { throw new Error("Should not be called"); };
-  await RemoteSettings.pollChanges();
-  // Last update is overwritten
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-
-  // Simulate a server error.
-  function simulateErrorResponse(request, response) {
-    response.setHeader("Date", (new Date(3000)).toUTCString());
-    response.setHeader("Content-Type", "application/json; charset=UTF-8");
-    response.write(JSON.stringify({
-      code: 503,
-      errno: 999,
-      error: "Service Unavailable",
-    }));
-    response.setStatusLine(null, 503, "Service Unavailable");
-  }
-  server.registerPathHandler(changesPath, simulateErrorResponse);
+}
 
-  // pollChanges() fails with adequate error and no notification.
-  let error;
-  notificationObserved = false;
-  Services.obs.addObserver(certblockObserver, "remote-settings-changes-polled");
-  try {
-    await RemoteSettings.pollChanges();
-  } catch (e) {
-    error = e;
-  }
-  Assert.ok(!notificationObserved, "a notification should not have been observed");
-  Assert.ok(/Polling for changes failed/.test(error.message));
-  // When an error occurs, last update was not overwritten (see Date header above).
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-  // check negative clock skew times
-
-  // set to a time in the future
-  server.registerPathHandler(changesPath, handleResponse.bind(null, Date.now() + 10000));
-
-  await RemoteSettings.pollChanges();
-
-  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
-  // we previously set the serverTime to Date.now() + 10000 ms past epoch
-  Assert.ok(clockDifference <= 0 && clockDifference >= -10);
-
-  //
-  // Backoff
-  //
-  function simulateBackoffResponse(request, response) {
-    response.setHeader("Content-Type", "application/json; charset=UTF-8");
-    response.setHeader("Backoff", "10");
-    response.write(JSON.stringify({data: []}));
+function serveChangesEntries(serverTime, entries) {
+  return (request, response) => {
     response.setStatusLine(null, 200, "OK");
-  }
-  server.registerPathHandler(changesPath, simulateBackoffResponse);
-  // First will work.
-  await RemoteSettings.pollChanges();
-  // Second will fail because we haven't waited.
-  try {
-    await RemoteSettings.pollChanges();
-    // The previous line should have thrown an error.
-    Assert.ok(false);
-  } catch (e) {
-    Assert.ok(/Server is asking clients to back off; retry in \d+s./.test(e.message));
-  }
-  // Once backoff time has expired, polling for changes can start again.
-  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
-  Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, `${Date.now() - 1000}`);
-  await RemoteSettings.pollChanges();
-  // Backoff tracking preference was cleared.
-  Assert.ok(!Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF));
-
-
-  // Simulate a network error (to check telemetry report).
-  Services.prefs.setCharPref(PREF_SETTINGS_SERVER, "http://localhost:42/v1");
-  try {
-    await RemoteSettings.pollChanges();
-  } catch (e) {}
-
-  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
-  // ensure that we've accumulated the correct telemetry
-  const expectedIncrements = {
-    [UptakeTelemetry.STATUS.UP_TO_DATE]: 4,
-    [UptakeTelemetry.STATUS.SUCCESS]: 1,
-    [UptakeTelemetry.STATUS.BACKOFF]: 1,
-    [UptakeTelemetry.STATUS.SERVER_ERROR]: 1,
-    [UptakeTelemetry.STATUS.NETWORK_ERROR]: 1,
-    [UptakeTelemetry.STATUS.UNKNOWN_ERROR]: 0,
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", (new Date(serverTime)).toUTCString());
+    if (entries.length) {
+      response.setHeader("ETag", `"${entries[0].last_modified}"`);
+    }
+    response.write(JSON.stringify({"data": entries}));
   };
-  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
-});
+}
 
 function run_test() {
   // Set up an HTTP Server
   server = new HttpServer();
   server.start(-1);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(function() { });
   });
 }
 
-// get a response for a given request from sample data
-function getSampleResponse(req, port) {
-  const responses = {
-    "GET:/v1/buckets/monitor/collections/changes/records?": {
-      "sampleHeaders": [
-        "Content-Type: application/json; charset=UTF-8",
-        "ETag: \"1100\""
-      ],
-      "status": {status: 200, statusText: "OK"},
-      "responseBody": JSON.stringify({"data": [{
-        "host": "localhost",
-        "last_modified": 1100,
-        "bucket": "test-bucket-aurora",
-        "id": "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
-        "collection": "test-collection"
-      }, {
-        "host": "localhost",
-        "last_modified": 1000,
-        "bucket": "test-bucket",
-        "id": "254cbb9e-6888-4d9f-8e60-58b74faa8778",
-        "collection": "test-collection"
-      }]})
+add_task(clear_state);
+
+add_task(async function test_check_success() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const serverTime = 8000;
+
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(serverTime, [{
+    id: "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
+    last_modified: 1100,
+    host: "localhost",
+    bucket: "some-other-bucket",
+    collection: "test-collection"
+  }, {
+    id: "254cbb9e-6888-4d9f-8e60-58b74faa8778",
+    last_modified: 1000,
+    host: "localhost",
+    bucket: "test-bucket",
+    collection: "test-collection"
+  }]));
+
+  // add a test kinto client that will respond to lastModified information
+  // for a collection called 'test-collection'
+  let maybeSyncCalled = false;
+  const c = RemoteSettings("test-collection", {
+    bucketName: "test-bucket",
+  });
+  c.maybeSync = () => { maybeSyncCalled = true; };
+
+  // Ensure that the remote-settings-changes-polled notification works
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
+    }
+  };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+
+  await RemoteSettings.pollChanges();
+
+  // It didn't fail, hence we are sure that the unknown collection ``some-other-bucket/test-collection``
+  // was ignored, otherwise it would have tried to reach the network.
+
+  Assert.ok(maybeSyncCalled, "maybeSync was called");
+  Assert.ok(notificationObserved, "a notification should have been observed");
+  // Last timestamp was saved. An ETag header value is a quoted string.
+  Assert.equal(Services.prefs.getCharPref(PREF_LAST_ETAG), "\"1100\"");
+  // check the last_update is updated
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000);
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SUCCESS]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_check_up_to_date() {
+  // Simulate a poll with up-to-date collection.
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  const serverTime = 4000;
+  function server304(request, response) {
+    if (request.hasHeader("if-none-match") && request.getHeader("if-none-match") == "\"1100\"") {
+      response.setHeader("Date", (new Date(serverTime)).toUTCString());
+      response.setStatusLine(null, 304, "Service Not Modified");
+    }
+  }
+  server.registerPathHandler(CHANGES_PATH, server304);
+
+  Services.prefs.setCharPref(PREF_LAST_ETAG, '"1100"');
+
+  // Ensure that the remote-settings-changes-polled notification is sent.
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
+    }
+  };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+
+  // If server has no change, a 304 is received, maybeSync() is not called.
+  let maybeSyncCalled = false;
+  const c = RemoteSettings("test-collection", {
+    bucketName: "test-bucket",
+  });
+  c.maybeSync = () => { maybeSyncCalled = true; };
+
+  await RemoteSettings.pollChanges();
+
+  Assert.ok(notificationObserved, "a notification should have been observed");
+  Assert.ok(!maybeSyncCalled, "maybeSync should not be called");
+  // Last update is overwritten
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000);
+
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.UP_TO_DATE]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_server_error() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  // Simulate a server error.
+  function simulateErrorResponse(request, response) {
+    response.setHeader("Date", (new Date(3000)).toUTCString());
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.write(JSON.stringify({
+      code: 503,
+      errno: 999,
+      error: "Service Unavailable",
+    }));
+    response.setStatusLine(null, 503, "Service Unavailable");
+  }
+  server.registerPathHandler(CHANGES_PATH, simulateErrorResponse);
+
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
     }
   };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 42);
 
-  if (req.hasHeader("if-none-match") && req.getHeader("if-none-match", "") == "\"1100\"")
-    return {sampleHeaders: [], status: {status: 304, statusText: "Not Modified"}, responseBody: ""};
+  // pollChanges() fails with adequate error and no notification.
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  Assert.ok(!notificationObserved, "a notification should not have been observed");
+  Assert.ok(/Polling for changes failed/.test(error.message));
+  // When an error occurs, last update was not overwritten.
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 42);
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SERVER_ERROR]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_check_clockskew_is_updated() {
+  const serverTime = 2000;
+
+  function serverResponse(request, response) {
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", (new Date(serverTime)).toUTCString());
+    response.write(JSON.stringify({data: []}));
+    response.setStatusLine(null, 200, "OK");
+  }
+  server.registerPathHandler(CHANGES_PATH, serverResponse);
+
+  let startTime = Date.now();
+
+  await RemoteSettings.pollChanges();
+
+  // How does the clock difference look?
+  let endTime = Date.now();
+  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to 2 (seconds past epoch)
+  Assert.ok(clockDifference <= endTime / 1000
+              && clockDifference >= Math.floor(startTime / 1000) - (serverTime / 1000));
+
+  // check negative clock skew times
+  // set to a time in the future
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(Date.now() + 10000, []));
+
+  await RemoteSettings.pollChanges();
+
+  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to Date.now() + 10000 ms past epoch
+  Assert.ok(clockDifference <= 0 && clockDifference >= -10);
+});
+add_task(clear_state);
+
+
+add_task(async function test_backoff() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  function simulateBackoffResponse(request, response) {
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Backoff", "10");
+    response.write(JSON.stringify({data: []}));
+    response.setStatusLine(null, 200, "OK");
+  }
+  server.registerPathHandler(CHANGES_PATH, simulateBackoffResponse);
+
+  // First will work.
+  await RemoteSettings.pollChanges();
+  // Second will fail because we haven't waited.
+  try {
+    await RemoteSettings.pollChanges();
+    // The previous line should have thrown an error.
+    Assert.ok(false);
+  } catch (e) {
+    Assert.ok(/Server is asking clients to back off; retry in \d+s./.test(e.message));
+  }
+
+  // Once backoff time has expired, polling for changes can start again.
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(12000, [{
+    id: "6a733d4a-601e-11e8-837a-0f85257529a1",
+    last_modified: 1300,
+    host: "localhost",
+    bucket: "some-bucket",
+    collection: "some-collection"
+  }]));
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, `${Date.now() - 1000}`);
+
+  await RemoteSettings.pollChanges();
+
+  // Backoff tracking preference was cleared.
+  Assert.ok(!Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF));
 
-  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
-         responses[req.method];
-}
+  // Ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SUCCESS]: 1,
+    [UptakeTelemetry.STATUS.UP_TO_DATE]: 1,
+    [UptakeTelemetry.STATUS.BACKOFF]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_network_error() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  // Simulate a network error (to check telemetry report).
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER, "http://localhost:42/v1");
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {}
+
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.NETWORK_ERROR]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_syncs_clients_with_local_database() {
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(42000, [{
+    id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844",
+    last_modified: 10000,
+    host: "localhost",
+    bucket: "main",
+    collection: "some-unknown"
+  }, {
+    id: "39f57e4e-6023-11e8-8b74-77c8dedfb389",
+    last_modified: 9000,
+    host: "localhost",
+    bucket: "blocklists",
+    collection: "addons"
+  }, {
+    id: "9a594c1a-601f-11e8-9c8a-33b2239d9113",
+    last_modified: 8000,
+    host: "localhost",
+    bucket: "main",
+    collection: "recipes"
+  }]));
+
+  // This simulates what remote-settings would do when initializing a local database.
+  // We don't want to instantiate a client using the RemoteSettings() API
+  // since we want to test «unknown» clients that have a local database.
+  await (new Kinto.adapters.IDB("blocklists/addons")).saveLastModified(42);
+  await (new Kinto.adapters.IDB("main/recipes")).saveLastModified(43);
+
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  // The `main/some-unknown` should be skipped because it has no local database.
+  // The `blocklists/addons` should be skipped because it is not the main bucket.
+  // The `recipes` has a local database, and should cause a network error because the test
+  // does not setup the server to receive the requests of `maybeSync()`.
+  Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync");
+  Assert.equal(error.details.collection, "recipes");
+});
+add_task(clear_state);
+
+
+add_task(async function test_syncs_clients_with_local_dump() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship remote settings dumps on Android (see package-manifest).
+    return;
+  }
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(42000, [{
+    id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844",
+    last_modified: 10000,
+    host: "localhost",
+    bucket: "main",
+    collection: "some-unknown"
+  }, {
+    id: "39f57e4e-6023-11e8-8b74-77c8dedfb389",
+    last_modified: 9000,
+    host: "localhost",
+    bucket: "blocklists",
+    collection: "addons"
+  }, {
+    id: "9a594c1a-601f-11e8-9c8a-33b2239d9113",
+    last_modified: 8000,
+    host: "localhost",
+    bucket: "main",
+    collection: "tippytop"
+  }]));
+
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  // The `main/some-unknown` should be skipped because it has no dump.
+  // The `blocklists/addons` should be skipped because it is not the main bucket.
+  // The `tippytop` has a dump, and should cause a network error because the test
+  // does not setup the server to receive the requests of `maybeSync()`.
+  Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync");
+  Assert.equal(error.details.collection, "tippytop");
+});
+add_task(clear_state);