--- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
@@ -41,24 +41,28 @@ function collectionRecordsPath(collectio
}
class KintoServer {
constructor() {
// Set up an HTTP Server
this.httpServer = new HttpServer();
this.httpServer.start(-1);
- // Map<CollectionId, Set<Object>> corresponding to the data in the
- // Kinto server
- this.collections = new Map();
+ // Set<Object> corresponding to records that might be served.
+ // The format of these objects is defined in the documentation for #addRecord.
+ this.records = [];
+
+ // Collections that we have set up access to (see `installCollection`).
+ this.collections = new Set();
// ETag to serve with responses
this.etag = 1;
this.port = this.httpServer.identity.primaryPort;
+
// POST requests we receive from the client go here
this.posts = [];
// DELETEd buckets will go here.
this.deletedBuckets = [];
// Anything in here will force the next POST to generate a conflict
this.conflicts = [];
this.installConfigPath();
@@ -148,28 +152,29 @@ class KintoServer {
status: 201, // FIXME -- only for new posts??
headers: {"ETag": 3000}, // FIXME???
body: oneBody,
};
}),
};
if (this.conflicts.length > 0) {
- const {collectionId, encrypted} = this.conflicts.shift();
- this.collections.get(collectionId).add(encrypted);
+ const nextConflict = this.conflicts.shift();
+ this.records.push(nextConflict);
+ const {data} = nextConflict;
dump(`responding with etag ${this.etag}\n`);
postResponse = {
responses: body.requests.map(req => {
return {
path: req.path,
status: 412,
headers: {"ETag": this.etag}, // is this correct??
body: {
details: {
- existing: encrypted,
+ existing: data,
},
},
};
}),
};
}
response.write(JSON.stringify(postResponse));
@@ -187,54 +192,95 @@ class KintoServer {
installCatchAll() {
this.httpServer.registerPathHandler("/", (request, response) => {
dump(`got request: ${request.method}:${request.path}?${request.queryString}\n`);
dump(`${CommonUtils.readBytesFromInputStream(request.bodyInputStream)}\n`);
});
}
+ /**
+ * Add a record to those that can be served by this server.
+ *
+ * @param {Object} properties An object describing the record that
+ * should be served. The properties of this object are:
+ * - collectionId {string} This record should only be served if a
+ * request is for this collection.
+ * - predicate {Function} If present, this record should only be served if the
+ * predicate returns true. The predicate will be called with
+ * {request: Request, response: Response, since: number, server: KintoServer}.
+ * - data {string} The record to serve.
+ * - conflict {boolean} If present and true, this record is added to
+ * "conflicts" and won't be served, but will cause a conflict on
+ * the next push.
+ */
+ addRecord(properties) {
+ if (!properties.conflict) {
+ this.records.push(properties);
+ } else {
+ this.conflicts.push(properties);
+ }
+
+ this.installCollection(properties.collectionId);
+ }
+
+ /**
+ * Tell the server to set up a route for this collection.
+ *
+ * This will automatically be called for any collection to which you `addRecord`.
+ *
+ * @param {string} collectionId the collection whose route we
+ * should set up.
+ */
installCollection(collectionId) {
- this.collections.set(collectionId, new Set());
-
+ if (this.collections.has(collectionId)) {
+ return;
+ }
+ this.collections.add(collectionId);
const remoteRecordsPath = "/v1" + collectionRecordsPath(encodeURIComponent(collectionId));
+ this.httpServer.registerPathHandler(remoteRecordsPath, this.handleGetRecords.bind(this, collectionId));
+ }
- function handleGetRecords(request, response) {
- if (request.method != "GET") {
- do_throw(`only GET is supported on ${remoteRecordsPath}`);
+ handleGetRecords(collectionId, request, response) {
+ if (request.method != "GET") {
+ do_throw(`only GET is supported on ${request.path}`);
+ }
+
+ let sinceMatch = request.queryString.match(/(^|&)_since=(\d+)/);
+ let since = sinceMatch && parseInt(sinceMatch[2], 10);
+
+ response.setStatusLine(null, 200, "OK");
+ response.setHeader("Content-Type", "application/json; charset=UTF-8");
+ response.setHeader("Date", (new Date()).toUTCString());
+ response.setHeader("ETag", this.etag.toString());
+
+ const records = this.records.filter(properties => {
+ if (properties.collectionId != collectionId) {
+ return false;
}
- response.setStatusLine(null, 200, "OK");
- response.setHeader("Content-Type", "application/json; charset=UTF-8");
- response.setHeader("Date", (new Date()).toUTCString());
- response.setHeader("ETag", this.etag.toString());
-
- const records = this.collections.get(collectionId);
- // Can't JSON a Set directly, so convert to Array
- let data = Array.from(records);
- if (request.queryString.includes("_since=")) {
- data = data.filter(r => !(r._inPast || false));
- }
-
- // Remove records that we only needed to serve once.
- // FIXME: come up with a more coherent idea of time here.
- // See bug 1321570.
- for (const record of records) {
- if (record._onlyOnce) {
- records.delete(record);
+ if (properties.predicate) {
+ const predAllowed = properties.predicate({
+ request: request,
+ response: response,
+ since: since,
+ server: this,
+ });
+ if (!predAllowed) {
+ return false;
}
}
- const body = JSON.stringify({
- "data": data,
- });
- response.write(body);
- }
+ return true;
+ }).map(properties => properties.data);
- this.httpServer.registerPathHandler(remoteRecordsPath, handleGetRecords.bind(this));
+ const body = JSON.stringify({
+ "data": records,
+ });
+ response.write(body);
}
installDeleteBucket() {
this.httpServer.registerPrefixHandler("/v1/buckets/", (request, response) => {
if (request.method != "DELETE") {
dump(`got a non-delete action on bucket: ${request.method} ${request.path}\n`);
return;
}
@@ -242,100 +288,75 @@ class KintoServer {
const noPrefix = request.path.slice("/v1/buckets/".length);
const [bucket, afterBucket] = noPrefix.split("/", 1);
if (afterBucket && afterBucket != "") {
dump(`got a delete for a non-bucket: ${request.method} ${request.path}\n`);
}
this.deletedBuckets.push(bucket);
// Fake like this actually deletes the records.
- for (const [, set] of this.collections) {
- set.clear();
- }
+ this.records = [];
response.write(JSON.stringify({
data: {
deleted: true,
last_modified: 1475161309026,
id: "b09f1618-d789-302d-696e-74ec53ee18a8", // FIXME
},
}));
});
}
// Utility function to install a keyring at the start of a test.
- installKeyRing(fxaService, keysData, salts, etag, {conflict = false} = {}) {
- this.installCollection("storage-sync-crypto");
+ installKeyRing(fxaService, keysData, salts, etag, properties) {
const keysRecord = {
"id": "keys",
"keys": keysData,
"salts": salts,
"last_modified": etag,
};
this.etag = etag;
- const methodName = conflict ? "encryptAndAddRecordWithConflict" : "encryptAndAddRecord";
- this[methodName](new KeyRingEncryptionRemoteTransformer(fxaService),
- "storage-sync-crypto", keysRecord);
- }
-
- // Add an already-encrypted record.
- addRecord(collectionId, record) {
- this.collections.get(collectionId).add(record);
- }
-
- // Add a record that is only served if no `_since` is present.
- //
- // Since in real life, Kinto only serves a record as part of a
- // changes feed if `_since` is before the record's modification
- // time, this can be helpful to test certain kinds of syncing logic.
- //
- // FIXME: tracking of "time" in this mock server really needs to be
- // implemented correctly rather than these hacks. See bug 1321570.
- addRecordInPast(collectionId, record) {
- record._inPast = true;
- this.addRecord(collectionId, record);
+ const transformer = new KeyRingEncryptionRemoteTransformer(fxaService);
+ this.encryptAndAddRecord(transformer, Object.assign({}, properties, {
+ collectionId: "storage-sync-crypto",
+ data: keysRecord,
+ }));
}
- encryptAndAddRecord(transformer, collectionId, record) {
- return transformer.encode(record).then(encrypted => {
- this.addRecord(collectionId, encrypted);
+ encryptAndAddRecord(transformer, properties) {
+ return transformer.encode(properties.data).then(encrypted => {
+ this.addRecord(Object.assign({}, properties, {data: encrypted}));
});
}
- // Like encryptAndAddRecord, but add a flag that will only serve
- // this record once.
- //
- // Since in real life, Kinto only serves a record as part of a changes feed
- // once, this can be useful for testing complicated syncing logic.
- //
- // FIXME: This kind of logic really needs to be subsumed into some
- // more-realistic tracking of "time" (simulated by etags). See bug 1321570.
- encryptAndAddRecordOnlyOnce(transformer, collectionId, record) {
- return transformer.encode(record).then(encrypted => {
- encrypted._onlyOnce = true;
- this.addRecord(collectionId, encrypted);
- });
- }
-
- // Conflicts block the next push and then appear in the collection specified.
- encryptAndAddRecordWithConflict(transformer, collectionId, record) {
- return transformer.encode(record).then(encrypted => {
- this.conflicts.push({collectionId, encrypted});
- });
- }
-
- clearCollection(collectionId) {
- this.collections.get(collectionId).clear();
- }
-
stop() {
this.httpServer.stop(() => { });
}
}
+/**
+ * Predicate that represents a record appearing at some time.
+ * Requests with "_since" before this time should see this record,
+ * unless the server itself isn't at this time yet (etag is before
+ * this time).
+ *
+ * Requests with _since after this time shouldn't see this record any
+ * more, since it hasn't changed after this time.
+ *
+ * @param {int} startTime the etag at which time this record should
+ * start being available (and thus, the predicate should start
+ * returning true)
+ * @returns {Function}
+ */
+function appearsAt(startTime) {
+ return function({since, server}) {
+ return since < startTime && startTime < server.etag;
+ };
+}
+
// Run a block of code with access to a KintoServer.
function* withServer(f) {
let server = new KintoServer();
// Point the sync.storage client to use the test server we've just started.
Services.prefs.setCharPref("webextensions.storage.sync.serverURL",
`http://localhost:${server.port}/v1`);
try {
yield* f(server);
@@ -521,17 +542,21 @@ add_task(function* ensureCanSync_posts_n
ok(oldSalt, `salts object should have a salt for ${extensionId}`);
// Try adding another key to make sure that the first post was
// OK, even on a new profile.
yield extensionStorageSync.cryptoCollection._clear();
server.clearPosts();
// Restore the first posted keyring, but add a last_modified date
const firstPostedKeyring = Object.assign({}, post.body.data, {last_modified: server.etag});
- server.addRecordInPast("storage-sync-crypto", firstPostedKeyring);
+ server.addRecord({
+ data: firstPostedKeyring,
+ collectionId: "storage-sync-crypto",
+ predicate: appearsAt(250),
+ });
const extensionId2 = uuid();
newKeys = yield extensionStorageSync.ensureCanSync([extensionId2]);
ok(newKeys.hasKeysFor([extensionId]), `didn't forget key for ${extensionId}`);
ok(newKeys.hasKeysFor([extensionId2]), `new key generated for ${extensionId2}`);
posts = server.getPosts();
equal(posts.length, 1);
const newPost = posts[posts.length - 1];
@@ -560,56 +585,61 @@ add_task(function* ensureCanSync_pulls_k
DEFAULT_KEY.generateRandom();
const RANDOM_KEY = new BulkKeyBundle(extensionId);
RANDOM_KEY.generateRandom();
yield* withContextAndServer(function* (context, server) {
yield* withSignedInUser(loggedInUser, function* (extensionStorageSync, fxaService) {
// FIXME: generating a random salt probably shouldn't require a CryptoCollection?
const cryptoCollection = new CryptoCollection(fxaService);
const RANDOM_SALT = cryptoCollection.getNewSalt();
+ yield extensionStorageSync.cryptoCollection._clear();
const keysData = {
"default": DEFAULT_KEY.keyPairB64,
"collections": {
[extensionId]: RANDOM_KEY.keyPairB64,
},
};
const saltData = {
[extensionId]: RANDOM_SALT,
};
- server.installKeyRing(fxaService, keysData, saltData, 999);
+ server.installKeyRing(fxaService, keysData, saltData, 950, {
+ predicate: appearsAt(900),
+ });
let collectionKeys = yield extensionStorageSync.ensureCanSync([extensionId]);
assertKeyRingKey(collectionKeys, extensionId, RANDOM_KEY);
let posts = server.getPosts();
equal(posts.length, 0,
"ensureCanSync shouldn't push when the server keyring has the right key");
// Another client generates a key for extensionId2
const newKey = new BulkKeyBundle(extensionId2);
newKey.generateRandom();
keysData.collections[extensionId2] = newKey.keyPairB64;
saltData[extensionId2] = cryptoCollection.getNewSalt();
- server.clearCollection("storage-sync-crypto");
- server.installKeyRing(fxaService, keysData, saltData, 1000);
+ server.installKeyRing(fxaService, keysData, saltData, 1050, {
+ predicate: appearsAt(1000),
+ });
let newCollectionKeys = yield extensionStorageSync.ensureCanSync([extensionId, extensionId2]);
assertKeyRingKey(newCollectionKeys, extensionId2, newKey);
assertKeyRingKey(newCollectionKeys, extensionId, RANDOM_KEY,
`ensureCanSync shouldn't lose the old key for ${extensionId}`);
posts = server.getPosts();
equal(posts.length, 0, "ensureCanSync shouldn't push when updating keys");
// Another client generates a key, but not a salt, for extensionOnlyKey
const onlyKey = new BulkKeyBundle(extensionOnlyKey);
onlyKey.generateRandom();
keysData.collections[extensionOnlyKey] = onlyKey.keyPairB64;
- server.clearCollection("storage-sync-crypto");
- server.installKeyRing(fxaService, keysData, saltData, 1001);
+ server.installKeyRing(fxaService, keysData, saltData, 1150, {
+ predicate: appearsAt(1100),
+ });
let withNewKey = yield extensionStorageSync.ensureCanSync([extensionId, extensionOnlyKey]);
dump(`got ${JSON.stringify(withNewKey.asWBO().cleartext)}\n`);
assertKeyRingKey(withNewKey, extensionOnlyKey, onlyKey);
assertKeyRingKey(withNewKey, extensionId, RANDOM_KEY,
`ensureCanSync shouldn't lose the old key for ${extensionId}`);
posts = server.getPosts();
@@ -618,18 +648,19 @@ add_task(function* ensureCanSync_pulls_k
// We don't a priori know what the new salt is
dump(`${JSON.stringify(withNewKeyRecord)}\n`);
ok(withNewKeyRecord.salts[extensionOnlyKey],
`ensureCanSync should generate a salt for an extension that only had a key`);
// Another client generates a key, but not a salt, for extensionOnlyKey
const newSalt = cryptoCollection.getNewSalt();
saltData[extensionOnlySalt] = newSalt;
- server.clearCollection("storage-sync-crypto");
- server.installKeyRing(fxaService, keysData, saltData, 1002);
+ server.installKeyRing(fxaService, keysData, saltData, 1250, {
+ predicate: appearsAt(1200),
+ });
let withOnlySaltKey = yield extensionStorageSync.ensureCanSync([extensionId, extensionOnlySalt]);
assertKeyRingKey(withOnlySaltKey, extensionId, RANDOM_KEY,
`ensureCanSync shouldn't lose the old key for ${extensionId}`);
// We don't a priori know what the new key is
ok(withOnlySaltKey.hasKeysFor([extensionOnlySalt]),
`ensureCanSync generated a key for an extension that only had a salt`);
@@ -757,29 +788,19 @@ add_task(function* checkSyncKeyRing_over
const extensionId = uuid();
let extensionKey;
yield* withSyncContext(function* (context) {
yield* withServer(function* (server) {
// The old device has this kB, which is very similar to the
// current kB but with the last f changed to an e.
const NOVEL_KB = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdee";
const oldUser = Object.assign({}, loggedInUser, {kB: NOVEL_KB});
- server.installCollection("storage-sync-crypto");
server.installDeleteBucket();
- server.etag = 765;
yield* withSignedInUser(oldUser, function* (extensionStorageSync, fxaService) {
- const transformer = new KeyRingEncryptionRemoteTransformer(fxaService);
- const FAKE_KEYRING = {
- id: "keys",
- keys: {},
- salts: {},
- uuid: "abcd",
- kbHash: "abcd",
- };
- yield server.encryptAndAddRecord(transformer, "storage-sync-crypto", FAKE_KEYRING);
+ yield server.installKeyRing(fxaService, {}, {}, 765);
});
// Now we have this new user with a different kB.
yield* withSignedInUser(loggedInUser, function* (extensionStorageSync, fxaService) {
yield extensionStorageSync.cryptoCollection._clear();
// Do an `ensureCanSync` to generate some keys.
// This will try to sync, notice that the record is
@@ -879,19 +900,22 @@ add_task(function* checkSyncKeyRing_flus
// this test verifies that just changing the UUID is enough.
const newKeyRingData = Object.assign({}, body, {
uuid: "abcd",
// Technically, last_modified should be served outside the
// object, but the transformer will pass it through in
// either direction, so this is OK.
last_modified: 765,
});
- server.clearCollection("storage-sync-crypto");
- server.etag = 765;
- yield server.encryptAndAddRecordOnlyOnce(transformer, "storage-sync-crypto", newKeyRingData);
+ server.etag = 1000;
+ yield server.encryptAndAddRecord(transformer, {
+ collectionId: "storage-sync-crypto",
+ data: newKeyRingData,
+ predicate: appearsAt(800),
+ });
// Fake adding another extension just so that the keyring will
// really get synced.
const newExtension = uuid();
const newKeyRing = yield extensionStorageSync.ensureCanSync([newExtension]);
// This should have detected the UUID change and flushed everything.
// The keyring should, however, be the same, since we just
@@ -936,22 +960,26 @@ add_task(function* test_storage_sync_pul
let calls = [];
yield extensionStorageSync.addOnChangedListener(extension, function() {
calls.push(arguments);
}, context);
yield extensionStorageSync.ensureCanSync([extensionId]);
const collectionId = yield cryptoCollection.extensionIdToCollectionId(extensionId);
- server.installCollection(collectionId);
- yield server.encryptAndAddRecord(transformer, collectionId, {
- "id": "key-remote_2D_key",
- "key": "remote-key",
- "data": 6,
+ yield server.encryptAndAddRecord(transformer, {
+ collectionId,
+ data: {
+ "id": "key-remote_2D_key",
+ "key": "remote-key",
+ "data": 6,
+ },
+ predicate: appearsAt(850),
});
+ server.etag = 900;
yield extensionStorageSync.syncAll();
const remoteValue = (yield extensionStorageSync.get(extension, "remote-key", context))["remote-key"];
equal(remoteValue, 6,
"ExtensionStorageSync.get() returns value retrieved from sync");
equal(calls.length, 1,
"syncing calls on-changed listener");
@@ -961,21 +989,24 @@ add_task(function* test_storage_sync_pul
// Syncing again doesn't do anything
yield extensionStorageSync.syncAll();
equal(calls.length, 0,
"syncing again shouldn't call on-changed listener");
// Updating the server causes us to pull down the new value
server.etag = 1000;
- server.clearCollection(collectionId);
- yield server.encryptAndAddRecord(transformer, collectionId, {
- "id": "key-remote_2D_key",
- "key": "remote-key",
- "data": 7,
+ yield server.encryptAndAddRecord(transformer, {
+ collectionId,
+ data: {
+ "id": "key-remote_2D_key",
+ "key": "remote-key",
+ "data": 7,
+ },
+ predicate: appearsAt(950),
});
yield extensionStorageSync.syncAll();
const remoteValue2 = (yield extensionStorageSync.get(extension, "remote-key", context))["remote-key"];
equal(remoteValue2, 7,
"ExtensionStorageSync.get() returns value updated from sync");
equal(calls.length, 1,
@@ -1075,20 +1106,23 @@ add_task(function* test_storage_sync_pul
server.clearPosts();
let calls = [];
yield extensionStorageSync.addOnChangedListener(extension, function() {
calls.push(arguments);
}, context);
const transformer = new CollectionKeyEncryptionRemoteTransformer(new CryptoCollection(fxaService), extension.id);
- yield server.encryptAndAddRecord(transformer, collectionId, {
- "id": "key-my_2D_key",
- "data": 6,
- "_status": "deleted",
+ yield server.encryptAndAddRecord(transformer, {
+ collectionId,
+ data: {
+ "id": "key-my_2D_key",
+ "data": 6,
+ "_status": "deleted",
+ },
});
yield extensionStorageSync.syncAll();
const remoteValues = (yield extensionStorageSync.get(extension, "my-key", context));
ok(!remoteValues["my-key"],
"ExtensionStorageSync.get() shows value was deleted by sync");
equal(server.getPosts().length, 0,