Bug 1451050 - Generalize loading of packaged remote settings dumps r?Gijs draft
authorMathieu Leplatre <mathieu@mozilla.com>
Thu, 10 May 2018 15:27:21 +0200
changeset 795944 529d59964c2b9e93c59edcc4bb8a50618aed72e9
parent 794690 45ec8fd380dd2c308e79dbb396ca87f2ce9b3f9c
push id110128
push usermleplatre@mozilla.com
push dateWed, 16 May 2018 20:43:05 +0000
reviewersGijs
bugs1451050
milestone62.0a1
Bug 1451050 - Generalize loading of packaged remote settings dumps r?Gijs MozReview-Commit-ID: FwD92fataAy
browser/base/content/test/static/browser_all_files_referenced.js
browser/installer/allowed-dupes.mn
browser/installer/package-manifest.in
mobile/android/installer/package-manifest.in
services/blocklists/addons.json
services/blocklists/certificates.json
services/blocklists/gfx.json
services/blocklists/moz.build
services/blocklists/pins.json
services/blocklists/plugins.json
services/blocklists/readme.md
services/common/docs/RemoteSettings.rst
services/common/remote-settings.js
services/common/tests/unit/test_blocklist_clients.js
services/common/tests/unit/test_blocklist_targetapp_filter.js
services/common/tests/unit/test_remote_settings.js
services/common/tests/unit/xpcshell.ini
services/moz.build
services/settings/dumps/blocklists/addons.json
services/settings/dumps/blocklists/certificates.json
services/settings/dumps/blocklists/gfx.json
services/settings/dumps/blocklists/moz.build
services/settings/dumps/blocklists/plugins.json
services/settings/dumps/main/moz.build
services/settings/dumps/main/tippytop.json
services/settings/dumps/moz.build
services/settings/dumps/pinning/moz.build
services/settings/dumps/pinning/pins.json
services/settings/dumps/readme.md
services/settings/moz.build
--- a/browser/base/content/test/static/browser_all_files_referenced.js
+++ b/browser/base/content/test/static/browser_all_files_referenced.js
@@ -8,18 +8,19 @@
 // Slow on asan builds.
 requestLongerTimeout(5);
 
 var isDevtools = SimpleTest.harnessParameters.subsuite == "devtools";
 
 var gExceptionPaths = [
   "chrome://browser/content/defaultthemes/",
   "chrome://browser/locale/searchplugins/",
-  "resource://app/defaults/blocklists/",
-  "resource://app/defaults/pinning/",
+  "resource://app/defaults/settings/blocklists/",
+  "resource://app/defaults/settings/main/",
+  "resource://app/defaults/settings/pinning/",
   "resource://app/defaults/preferences/",
   "resource://gre/modules/commonjs/",
   "resource://gre/defaults/pref/",
 
   // These resources are referenced using relative paths from html files.
   "resource://payments/",
   "resource://normandy-content/shield-content-frame.js",
   "resource://normandy-content/shield-content-process.js",
--- a/browser/installer/allowed-dupes.mn
+++ b/browser/installer/allowed-dupes.mn
@@ -145,8 +145,11 @@ browser/chrome/browser/content/branding/
 browser/chrome/devtools/content/framework/dev-edition-promo/dev-edition-logo.png
 # Bug 1451016 - Nightly-only PaymentRequest & Form Autofill code sharing.
 browser/features/formautofill@mozilla.org/chrome/content/editAddress.xhtml
 browser/chrome/browser/res/payments/formautofill/editAddress.xhtml
 browser/features/formautofill@mozilla.org/chrome/content/editCreditCard.xhtml
 browser/chrome/browser/res/payments/formautofill/editCreditCard.xhtml
 browser/features/formautofill@mozilla.org/chrome/content/autofillEditForms.js
 browser/chrome/browser/res/payments/formautofill/autofillEditForms.js
+# Bug 1451050 - Remote settings empty dumps (will be populated with data eventually)
+browser/defaults/settings/pinning/pins.json
+browser/defaults/settings/main/tippytop.json
--- a/browser/installer/package-manifest.in
+++ b/browser/installer/package-manifest.in
@@ -444,18 +444,19 @@
 
 ; [Default Preferences]
 ; All the pref files must be part of base to prevent migration bugs
 @RESPATH@/browser/@PREF_DIR@/firefox.js
 @RESPATH@/browser/@PREF_DIR@/firefox-branding.js
 @RESPATH@/greprefs.js
 @RESPATH@/defaults/autoconfig/prefcalls.js
 @RESPATH@/browser/defaults/permissions
-@RESPATH@/browser/defaults/blocklists
-@RESPATH@/browser/defaults/pinning
+@RESPATH@/browser/defaults/settings/blocklists
+@RESPATH@/browser/defaults/settings/pinning
+@RESPATH@/browser/defaults/settings/main
 
 ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
 ; Technically this is an app pref file, but we are keeping it in the original
 ; gre location for now.
 @RESPATH@/defaults/pref/channel-prefs.js
 
 ; Services (gre) prefs
 @RESPATH@/defaults/pref/services-sync.js
--- a/mobile/android/installer/package-manifest.in
+++ b/mobile/android/installer/package-manifest.in
@@ -82,16 +82,18 @@
 [xpcom]
 @BINPATH@/package-name.txt
 
 [browser]
 ; [Base Browser Files]
 @BINPATH@/application.ini
 @BINPATH@/platform.ini
 @BINPATH@/blocklist.xml
+@BINPATH@/defaults/settings/blocklists/addons.json
+@BINPATH@/defaults/settings/blocklists/certificates.json
 
 ; [Components]
 @BINPATH@/components/components.manifest
 
 ; JavaScript components
 @BINPATH@/components/ConsoleAPI.manifest
 @BINPATH@/components/ConsoleAPIStorage.js
 @BINPATH@/components/NotificationStorage.js
--- a/services/common/docs/RemoteSettings.rst
+++ b/services/common/docs/RemoteSettings.rst
@@ -85,16 +85,28 @@ When an entry has a file attached to it,
 
     data.filter(d => d.attachment)
         .forEach(async ({ attachment: { url, filename, size } }) => {
           if (size < OS.freeDiskSpace) {
             await downloadLocally(url, filename);
           }
         });
 
+Initial data
+------------
+
+For newly created user profiles, the list of entries returned by the ``.get()`` method will be empty until the first synchronization happens.
+
+It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet. It will thus serve as the default dataset and also reduce the amount of data to be downloaded on the first synchronization.
+
+#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder
+#. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build``
+
+Now, when ``RemoteSettings("some-key").get()`` is called from an empty profile, the ``some-key.json`` file is going to be loaded before the results are returned.
+
 
 Uptake Telemetry
 ================
 
 Some :ref:`uptake telemetry <telemetry/collection/uptake>` is collected in order to monitor how remote settings are propagated.
 
 It is submitted to a single :ref:`keyed histogram <histogram-type-keyed>` whose id is ``UPTAKE_REMOTE_CONTENT_RESULT_1`` and the keys are prefixed with ``main/`` (eg. ``main/a-key`` in the above example).
 
--- a/services/common/remote-settings.js
+++ b/services/common/remote-settings.js
@@ -208,16 +208,31 @@ class RemoteSettingsClient {
    * @param  {Object} options.order   The order to apply   (default: `-last_modified`).
    * @return {Promise}
    */
   async get(options = {}) {
     // In Bug 1451031, we will do some jexl filtering to limit the list items
     // whose target is matched.
     const { filters = {}, order } = options;
     const c = await this.openCollection();
+
+    const timestamp = await c.db.getLastModified();
+    // If the local database was never synchronized, then we attempt to load
+    // a packaged JSON dump.
+    if (timestamp == null) {
+      try {
+        const { data } = await this._loadDumpFile();
+        await c.loadDump(data);
+      } catch (e) {
+        // Report but return an empty list since there will be no data anyway.
+        Cu.reportError(e);
+        return [];
+      }
+    }
+
     const { data } = await c.list({ filters, order });
     return this._filterEntries(data);
   }
 
   /**
    * Synchronize from Kinto server, if necessary.
    *
    * @param {int}  lastModified       the lastModified date (on the server) for
@@ -397,17 +412,17 @@ class RemoteSettingsClient {
   }
 
   /**
    * Load the the JSON file distributed with the release for this collection.
    */
   async _loadDumpFile() {
     // Replace OS specific path separator by / for URI.
     const { components: folderFile } = OS.Path.split(this.filename);
-    const fileURI = `resource://app/defaults/${folderFile.join("/")}`;
+    const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
     const response = await fetch(fileURI);
     if (!response.ok) {
       throw new Error(`Could not read from '${fileURI}'`);
     }
     // Will be rejected if JSON is invalid.
     return response.json();
   }
 
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -1,20 +1,23 @@
 const { Constructor: CC } = Components;
 
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 
 const BlocklistClients = ChromeUtils.import("resource://services-common/blocklist-clients.js", {});
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
 
+const IS_ANDROID = AppConstants.platform == "android";
+
 
 let gBlocklistClients;
 let server;
 
 async function readJSON(filepath) {
   const binaryData = await OS.File.read(filepath);
   const textData = (new TextDecoder()).decode(binaryData);
   return Promise.resolve(JSON.parse(textData));
@@ -94,24 +97,53 @@ function run_test() {
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
 
 add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is_empty() {
+  const november2016 = 1480000000000;
+
   for (let {client} of gBlocklistClients) {
+    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
+      // On Android we don't ship the dumps of plugins and gfx.
+      continue;
+    }
+
     // Test an empty db populates, but don't reach server (specified timestamp <= dump).
     await client.maybeSync(1, Date.now());
 
     // Verify the loaded data has status to synced:
     const collection = await client.openCollection();
     const { data: list } = await collection.list();
     equal(list[0]._status, "synced");
+
+    // Verify that the internal timestamp was updated.
+    const timestamp = await collection.db.getLastModified();
+    ok(timestamp > november2016, `Loaded dump of ${client.collectionName} has timestamp ${timestamp}`);
+  }
+});
+add_task(clear_state);
+
+add_task(async function test_initial_dump_is_loaded_when_using_get_on_empty_collection() {
+  for (let {client} of gBlocklistClients) {
+    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
+      // On Android we don't ship the dumps of plugins and gfx.
+      continue;
+    }
+    // Internal database is empty.
+    const collection = await client.openCollection();
+    const { data: list } = await collection.list();
+    equal(list.length, 0);
+
+    // Calling .get() will load the dump.
+    const afterLoaded = await client.get();
+    ok(afterLoaded.length > 0, `Loaded dump of ${client.collectionName} has ${afterLoaded.length} records`);
   }
 });
 add_task(clear_state);
 
 add_task(async function test_list_is_written_to_file_in_profile() {
   for (let {client, testData} of gBlocklistClients) {
     const filePath = OS.Path.join(OS.Constants.Path.profileDir, client.filename);
     const profFile = new FileUtils.File(filePath);
--- a/services/common/tests/unit/test_blocklist_targetapp_filter.js
+++ b/services/common/tests/unit/test_blocklist_targetapp_filter.js
@@ -11,16 +11,17 @@ async function clear_state() {
   await collection.clear();
 }
 
 async function createRecords(records) {
   const collection = await client.openCollection();
   for (const record of records) {
     await collection.create(record);
   }
+  collection.db.saveLastModified(42); // Simulate sync (and prevent load dump).
 }
 
 
 function run_test() {
   // This will initialize the remote settings clients for blocklists,
   // with their specific options etc.
   BlocklistClients.initialize();
   // Obtain one of the instantiated client for our tests.
--- a/services/common/tests/unit/test_remote_settings.js
+++ b/services/common/tests/unit/test_remote_settings.js
@@ -98,16 +98,27 @@ add_task(async function test_records_cha
 
   await client.maybeSync(2000, Date.now());
 
   const data = await client.get();
   equal(data[0].website, "https://some-website.com");
 });
 add_task(clear_state);
 
+add_task(async function test_default_records_come_from_a_local_dump_when_database_is_empty() {
+  // When collection is unknown, no dump is loaded, and there is no error.
+  let data = await RemoteSettings("some-unknown-key").get();
+  equal(data.length, 0);
+
+  // When collection has a dump in services/settings/dumps/{bucket}/{collection}.json
+  data = await RemoteSettings("certificates", { bucketName: "blocklists" }).get();
+  notEqual(data.length, 0);
+});
+add_task(clear_state);
+
 add_task(async function test_sync_event_provides_information_about_records() {
   const serverTime = Date.now();
 
   let eventData;
   client.on("sync", ({ data }) => eventData = data);
 
   await client.maybeSync(2000, serverTime - 1000);
   equal(eventData.current.length, 1);
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@@ -4,18 +4,18 @@ firefox-appdir = browser
 support-files =
   test_storage_adapter/**
   test_blocklist_signatures/**
 
 # Test load modules first so syntax failures are caught early.
 [test_load_modules.js]
 
 [test_blocklist_certificates.js]
-# Initial JSON data for blocklists are not shipped on Android.
-skip-if = (os == "android" || appname == "thunderbird")
+# Skip signature tests for Thunderbird (Bug 1341983).
+skip-if = appname == "thunderbird"
 tags = blocklist
 [test_blocklist_clients.js]
 tags = blocklist
 [test_blocklist_targetapp_filter.js]
 tags = blocklist
 [test_blocklist_pinning.js]
 tags = blocklist
 [test_remote_settings.js]
--- a/services/moz.build
+++ b/services/moz.build
@@ -5,18 +5,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files('moz.build'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
 DIRS += [
     'common',
     'crypto',
+    'settings',
 ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
     DIRS += [
         'fxaccounts',
-        'blocklists',
     ]
 
 if CONFIG['MOZ_SERVICES_SYNC']:
     DIRS += ['sync']
rename from services/blocklists/addons.json
rename to services/settings/dumps/blocklists/addons.json
rename from services/blocklists/certificates.json
rename to services/settings/dumps/blocklists/certificates.json
rename from services/blocklists/gfx.json
rename to services/settings/dumps/blocklists/gfx.json
new file mode 100644
--- /dev/null
+++ b/services/settings/dumps/blocklists/moz.build
@@ -0,0 +1,16 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Toolkit', 'Blocklisting')
+
+FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons.json',
+                                                    'certificates.json',
+                                                    'gfx.json',
+                                                    'plugins.json']
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'
rename from services/blocklists/plugins.json
rename to services/settings/dumps/blocklists/plugins.json
new file mode 100644
--- /dev/null
+++ b/services/settings/dumps/main/moz.build
@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_TARGET_FILES.defaults.settings.main += [
+    'tippytop.json',
+]
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'
new file mode 100644
--- /dev/null
+++ b/services/settings/dumps/main/tippytop.json
@@ -0,0 +1,1 @@
+{"data":[]}
\ No newline at end of file
rename from services/blocklists/moz.build
rename to services/settings/dumps/moz.build
--- a/services/blocklists/moz.build
+++ b/services/settings/dumps/moz.build
@@ -1,18 +1,9 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-with Files('**'):
-    BUG_COMPONENT = ('Toolkit', 'Blocklisting')
-
-FINAL_TARGET_FILES.defaults.blocklists += ['addons.json',
-                                           'certificates.json',
-                                           'gfx.json',
-                                           'plugins.json']
-
-FINAL_TARGET_FILES.defaults.pinning += ['pins.json']
-
-if CONFIG['MOZ_BUILD_APP'] == 'browser':
-    DIST_SUBDIR = 'browser'
+DIRS += [
+    'blocklists',
+    'main',
+    'pinning',
+]
new file mode 100644
--- /dev/null
+++ b/services/settings/dumps/pinning/moz.build
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_TARGET_FILES.defaults.settings.pinning += ['pins.json']
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'
rename from services/blocklists/pins.json
rename to services/settings/dumps/pinning/pins.json
rename from services/blocklists/readme.md
rename to services/settings/dumps/readme.md
new file mode 100644
--- /dev/null
+++ b/services/settings/moz.build
@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Firefox', 'Remote Settings Client')
+
+DIRS += [
+    'dumps',
+]