Bug 1451031 - Add JEXL filter support in Remote Settings r=mythmon draft
authorMathieu Leplatre <mathieu@mozilla.com>
Fri, 11 May 2018 17:09:44 +0200
changeset 798341 e030d29b915de8a1cbaa6f11f70231d1015febdf
parent 796870 11ee70f24ea52c4dc4f113593c288f4a6dc92c55
push id110719
push usermleplatre@mozilla.com
push dateTue, 22 May 2018 17:33:16 +0000
reviewersmythmon
bugs1451031
milestone62.0a1
Bug 1451031 - Add JEXL filter support in Remote Settings r=mythmon MozReview-Commit-ID: DwVdW1G3yZG
services/common/blocklist-clients.js
services/common/docs/RemoteSettings.rst
services/common/remote-settings.js
services/common/tests/unit/test_blocklist_clients.js
services/common/tests/unit/test_remote_settings_jexl_filters.js
services/common/tests/unit/xpcshell.ini
--- a/services/common/blocklist-clients.js
+++ b/services/common/blocklist-clients.js
@@ -6,18 +6,18 @@
 
 var EXPORTED_SYMBOLS = [
   "initialize",
 ];
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 
-ChromeUtils.defineModuleGetter(this, "RemoteSettings",
-                               "resource://services-common/remote-settings.js");
+ChromeUtils.defineModuleGetter(this, "RemoteSettings", "resource://services-common/remote-settings.js");
+ChromeUtils.defineModuleGetter(this, "jexlFilterFunc", "resource://services-common/remote-settings.js");
 
 const PREF_BLOCKLIST_BUCKET                  = "services.blocklist.bucket";
 const PREF_BLOCKLIST_ONECRL_COLLECTION       = "services.blocklist.onecrl.collection";
 const PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS  = "services.blocklist.onecrl.checked";
 const PREF_BLOCKLIST_ONECRL_SIGNER           = "services.blocklist.onecrl.signer";
 const PREF_BLOCKLIST_ADDONS_COLLECTION       = "services.blocklist.addons.collection";
 const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS  = "services.blocklist.addons.checked";
 const PREF_BLOCKLIST_ADDONS_SIGNER           = "services.blocklist.addons.signer";
@@ -127,27 +127,32 @@ async function updateJSONBlocklist(clien
   }
 }
 
 
 /**
  * This custom filter function is used to limit the entries returned
  * by `RemoteSettings("...").get()` depending on the target app information
  * defined on entries.
- *
- * When landing Bug 1451031, this function will have to check if the `entry`
- * has a JEXL attribute and rely on the JEXL filter function in priority.
- * The legacy target app mechanism will be kept in place for old entries.
  */
-async function targetAppFilter(entry, { appID, version: appVersion }) {
+async function targetAppFilter(entry, environment) {
+  // If the entry has JEXL filters, they should prevail.
+  // The legacy target app mechanism will be kept in place for old entries.
+  // See https://bugzilla.mozilla.org/show_bug.cgi?id=1463377
+  const { filters } = entry;
+  if (filters) {
+    return jexlFilterFunc(entry, environment);
+  }
+
   // Keep entries without target information.
   if (!("versionRange" in entry)) {
     return entry;
   }
 
+  const { appID, version: appVersion } = environment;
   const { versionRange } = entry;
 
   // Gfx blocklist has a specific versionRange object, which is not a list.
   if (!Array.isArray(versionRange)) {
     const { minVersion = "0", maxVersion = "*" } = versionRange;
     const matchesRange = (Services.vc.compare(appVersion, minVersion) >= 0 &&
                           Services.vc.compare(appVersion, maxVersion) <= 0);
     return matchesRange ? entry : null;
--- a/services/common/docs/RemoteSettings.rst
+++ b/services/common/docs/RemoteSettings.rst
@@ -98,16 +98,28 @@ For newly created user profiles, the lis
 It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet. It will thus serve as the default dataset and also reduce the amount of data to be downloaded on the first synchronization.
 
 #. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder
 #. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build``
 
 Now, when ``RemoteSettings("some-key").get()`` is called from an empty profile, the ``some-key.json`` file is going to be loaded before the results are returned.
 
 
+Targets and A/B testing
+=======================
+
+In order to deliver settings to subsets of the population, you can set targets on entries (platform, language, channel, version range, preferences values, samples, etc.) when editing records on the server.
+
+From the client API standpoint, this is completely transparent: the ``.get()`` method — as well as the event data — will always filter the entries on which the target matches.
+
+.. note::
+
+    The remote settings targets follow the same approach as the :ref:`Normandy recipe client <components/normandy>` (ie. JEXL filters),
+
+
 Uptake Telemetry
 ================
 
 Some :ref:`uptake telemetry <telemetry/collection/uptake>` is collected in order to monitor how remote settings are propagated.
 
 It is submitted to a single :ref:`keyed histogram <histogram-type-keyed>` whose id is ``UPTAKE_REMOTE_CONTENT_RESULT_1`` and the keys are prefixed with ``main/`` (eg. ``main/a-key`` in the above example).
 
 
--- a/services/common/remote-settings.js
+++ b/services/common/remote-settings.js
@@ -1,31 +1,35 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
-var EXPORTED_SYMBOLS = ["RemoteSettings"];
+var EXPORTED_SYMBOLS = [
+  "RemoteSettings",
+  "jexlFilterFunc"
+];
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 Cu.importGlobalProperties(["fetch"]);
 
 ChromeUtils.defineModuleGetter(this, "Kinto",
                                "resource://services-common/kinto-offline-client.js");
 ChromeUtils.defineModuleGetter(this, "KintoHttpClient",
                                "resource://services-common/kinto-http-client.js");
 ChromeUtils.defineModuleGetter(this, "CanonicalJSON",
                                "resource://gre/modules/CanonicalJSON.jsm");
 ChromeUtils.defineModuleGetter(this, "UptakeTelemetry",
                                "resource://services-common/uptake-telemetry.js");
 ChromeUtils.defineModuleGetter(this, "ClientEnvironmentBase",
                                "resource://gre/modules/components-utils/ClientEnvironment.jsm");
+ChromeUtils.defineModuleGetter(this, "FilterExpressions", "resource://normandy/lib/FilterExpressions.jsm");
 
 const PREF_SETTINGS_SERVER             = "services.settings.server";
 const PREF_SETTINGS_DEFAULT_BUCKET     = "services.settings.default_bucket";
 const PREF_SETTINGS_DEFAULT_SIGNER     = "services.settings.default_signer";
 const PREF_SETTINGS_VERIFY_SIGNATURE   = "services.settings.verify_signature";
 const PREF_SETTINGS_SERVER_BACKOFF     = "services.settings.server.backoff";
 const PREF_SETTINGS_CHANGES_PATH       = "services.settings.changes.path";
 const PREF_SETTINGS_LAST_UPDATE        = "services.settings.last_update_seconds";
@@ -57,16 +61,37 @@ function cacheProxy(target) {
 class ClientEnvironment extends ClientEnvironmentBase {
   static get appID() {
     // eg. Firefox is "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}".
     Services.appinfo.QueryInterface(Ci.nsIXULAppInfo);
     return Services.appinfo.ID;
   }
 }
 
+/**
+ * Default entry filtering function, in charge of excluding remote settings entries
+ * where the JEXL expression evaluates into a falsy value.
+ */
+async function jexlFilterFunc(entry, environment) {
+  const { filters } = entry;
+  if (!filters) {
+    return entry;
+  }
+  let result;
+  try {
+    const context = {
+      environment
+    };
+    result = await FilterExpressions.eval(filters, context);
+  } catch (e) {
+    Cu.reportError(e);
+  }
+  return result ? entry : null;
+}
+
 
 function mergeChanges(collection, localRecords, changes) {
   const records = {};
   // Local records by id.
   localRecords.forEach((record) => records[record.id] = collection.cleanLocalFields(record));
   // All existing records are replaced by the version from the server.
   changes.forEach((record) => records[record.id] = record);
 
@@ -145,17 +170,17 @@ async function fetchLatestChanges(url, l
   }
 
   return {changes, currentEtag, serverTimeMillis, backoffSeconds};
 }
 
 
 class RemoteSettingsClient {
 
-  constructor(collectionName, { bucketName, signerName, filterFunc, lastCheckTimePref }) {
+  constructor(collectionName, { bucketName, signerName, filterFunc = jexlFilterFunc, lastCheckTimePref }) {
     this.collectionName = collectionName;
     this.bucketName = bucketName;
     this.signerName = signerName;
     this.filterFunc = filterFunc;
     this._lastCheckTimePref = lastCheckTimePref;
 
     this._callbacks = new Map();
     this._callbacks.set("sync", []);
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -240,16 +240,67 @@ add_task(async function test_sync_event_
     // and the event current data should differ.
     const collection = await client.openCollection();
     const { data: internalData } = await collection.list();
     ok(internalData.length > current.length, `event current data for ${client.collectionName}`);
   }
 });
 add_task(clear_state);
 
+add_task(async function test_entries_are_filtered_when_jexl_filters_is_present() {
+  if (IS_ANDROID) {
+    // JEXL filters are not supported on Android.
+    // See https://bugzilla.mozilla.org/show_bug.cgi?id=1463502
+    return;
+  }
+
+  const records = [{
+      willMatch: true,
+    }, {
+      willMatch: true,
+      filters: null
+    }, {
+      willMatch: true,
+      filters: "1 == 1"
+    }, {
+      willMatch: false,
+      filters: "1 == 2"
+    }, {
+      willMatch: true,
+      filters: "1 == 1",
+      versionRange: [{
+        targetApplication: [{
+          guid: "some-guid"
+        }],
+      }]
+    }, {
+      willMatch: false,  // jexl prevails over versionRange.
+      filters: "1 == 2",
+      versionRange: [{
+        targetApplication: [{
+          guid: "xpcshell@tests.mozilla.org",
+          minVersion: "0",
+          maxVersion: "*",
+        }],
+      }]
+    }
+  ];
+  for (let {client} of gBlocklistClients) {
+    const collection = await client.openCollection();
+    for (const record of records) {
+      await collection.create(record);
+    }
+    await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
+    const list = await client.get();
+    equal(list.length, 4);
+    ok(list.every(e => e.willMatch));
+  }
+});
+add_task(clear_state);
+
 
 // get a response for a given request from sample data
 function getSampleResponse(req, port) {
   const responses = {
     "OPTIONS": {
       "sampleHeaders": [
         "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
         "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
new file mode 100644
--- /dev/null
+++ b/services/common/tests/unit/test_remote_settings_jexl_filters.js
@@ -0,0 +1,171 @@
+const { RemoteSettings } = ChromeUtils.import("resource://services-common/remote-settings.js", {});
+
+let client;
+
+async function createRecords(records) {
+  const collection = await client.openCollection();
+  await collection.clear();
+  for (const record of records) {
+    await collection.create(record);
+  }
+  await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
+}
+
+
+function run_test() {
+  client = RemoteSettings("some-key");
+
+  run_next_test();
+}
+
+add_task(async function test_returns_all_without_target() {
+  await createRecords([{
+    passwordSelector: "#pass-signin"
+  }, {
+    filters: null,
+  }, {
+    filters: "",
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+});
+
+add_task(async function test_filters_can_be_disabled() {
+  const c = RemoteSettings("no-jexl", { filterFunc: null });
+  const collection = await c.openCollection();
+  await collection.create({
+    filters: "1 == 2"
+  });
+  await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
+
+  const list = await c.get();
+  equal(list.length, 1);
+});
+
+add_task(async function test_returns_entries_where_jexl_is_true() {
+  await createRecords([{
+    willMatch: true,
+    filters: "1"
+  }, {
+    willMatch: true,
+    filters: "[42]"
+  }, {
+    willMatch: true,
+    filters: "1 == 2 || 1 == 1"
+  }, {
+    willMatch: true,
+    filters: 'environment.appID == "xpcshell@tests.mozilla.org"'
+  }, {
+    willMatch: false,
+    filters: "environment.version == undefined"
+  }, {
+    willMatch: true,
+    filters: "environment.unknown == undefined"
+  }, {
+    willMatch: false,
+    filters: "1 == 2"
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 5);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_ignores_entries_where_jexl_is_invalid() {
+  await createRecords([{
+    filters: "true === true"  // JavaScript Error: "Invalid expression token: ="
+  }, {
+    filters: "Objects.keys({}) == []" // Token ( (openParen) unexpected in expression
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 0);
+});
+
+add_task(async function test_support_of_date_filters() {
+  await createRecords([{
+    willMatch: true,
+    filters: '"1982-05-08"|date < "2016-03-22"|date'
+  }, {
+    willMatch: false,
+    filters: '"2000-01-01"|date < "1970-01-01"|date'
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 1);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_preferences_filters() {
+  await createRecords([{
+    willMatch: true,
+    filters: '"services.settings.last_etag"|preferenceValue == 42'
+  }, {
+    willMatch: true,
+    filters: '"services.settings.changes.path"|preferenceExists == true'
+  }, {
+    willMatch: true,
+    filters: '"services.settings.changes.path"|preferenceIsUserSet == false'
+  }, {
+    willMatch: true,
+    filters: '"services.settings.last_etag"|preferenceIsUserSet == true'
+  }]);
+
+  // Set a pref for the user.
+  Services.prefs.setIntPref("services.settings.last_etag", 42);
+
+  const list = await client.get();
+  equal(list.length, 4);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_intersect_operator() {
+  await createRecords([{
+    willMatch: true,
+    filters: '{foo: 1, bar: 2}|keys intersect ["foo"]'
+  }, {
+    willMatch: true,
+    filters: '(["a", "b"] intersect ["a", 1, 4]) == "a"'
+  }, {
+    willMatch: false,
+    filters: '(["a", "b"] intersect [3, 1, 4]) == "c"'
+  }, {
+    willMatch: true,
+    filters: `
+      [1, 2, 3]
+        intersect
+      [3, 4, 5]
+    `
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+  ok(list.every(e => e.willMatch));
+});
+
+add_task(async function test_support_of_samples() {
+  await createRecords([{
+    willMatch: true,
+    filters: '"always-true"|stableSample(1)'
+  }, {
+    willMatch: false,
+    filters: '"always-false"|stableSample(0)'
+  }, {
+    willMatch: true,
+    filters: '"turns-to-true-0"|stableSample(0.5)'
+  }, {
+    willMatch: false,
+    filters: '"turns-to-false-1"|stableSample(0.5)'
+  }, {
+    willMatch: true,
+    filters: '"turns-to-true-0"|bucketSample(0, 50, 100)'
+  }, {
+    willMatch: false,
+    filters: '"turns-to-false-1"|bucketSample(0, 50, 100)'
+  }]);
+
+  const list = await client.get();
+  equal(list.length, 3);
+  ok(list.every(e => e.willMatch));
+});
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@@ -17,16 +17,19 @@ tags = blocklist
 [test_blocklist_targetapp_filter.js]
 tags = blocklist
 [test_blocklist_pinning.js]
 tags = blocklist
 [test_remote_settings.js]
 tags = remote-settings blocklist
 [test_remote_settings_poll.js]
 tags = remote-settings blocklist
+[test_remote_settings_jexl_filters.js]
+skip-if = os == "android"
+tags = remote-settings
 
 [test_kinto.js]
 tags = blocklist
 [test_blocklist_signatures.js]
 tags = remote-settings blocklist
 [test_storage_adapter.js]
 tags = remote-settingsblocklist
 [test_storage_adapter_shutdown.js]