--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -41,17 +41,16 @@ MACH_MODULES = [
'python/mach_commands.py',
'python/mach/mach/commands/commandinfo.py',
'python/mach/mach/commands/settings.py',
'python/mozboot/mozboot/mach_commands.py',
'python/mozbuild/mozbuild/mach_commands.py',
'python/mozbuild/mozbuild/backend/mach_commands.py',
'python/mozbuild/mozbuild/compilation/codecomplete.py',
'python/mozbuild/mozbuild/frontend/mach_commands.py',
- 'services/common/tests/mach_commands.py',
'taskcluster/mach_commands.py',
'testing/awsy/mach_commands.py',
'testing/firefox-ui/mach_commands.py',
'testing/geckodriver/mach_commands.py',
'testing/mach_commands.py',
'testing/marionette/mach_commands.py',
'testing/mochitest/mach_commands.py',
'testing/mozharness/mach_commands.py',
deleted file mode 100644
--- a/services/common/modules-testing/storageserver.js
+++ /dev/null
@@ -1,1691 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/**
- * This file contains an implementation of the Storage Server in JavaScript.
- *
- * The server should not be used for any production purposes.
- */
-
-var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
-
-this.EXPORTED_SYMBOLS = [
- "ServerBSO",
- "StorageServerCallback",
- "StorageServerCollection",
- "StorageServer",
- "storageServerForUsers",
-];
-
-Cu.import("resource://testing-common/httpd.js");
-Cu.import("resource://services-common/async.js");
-Cu.import("resource://gre/modules/Log.jsm");
-Cu.import("resource://services-common/utils.js");
-
-const STORAGE_HTTP_LOGGER = "Services.Common.Test.Server";
-const STORAGE_API_VERSION = "2.0";
-
-// Use the same method that record.js does, which mirrors the server.
-function new_timestamp() {
- return Math.round(Date.now());
-}
-
-function isInteger(s) {
- let re = /^[0-9]+$/;
- return re.test(s);
-}
-
-function writeHttpBody(response, body) {
- if (!body) {
- return;
- }
-
- response.bodyOutputStream.write(body, body.length);
-}
-
-function sendMozSvcError(request, response, code) {
- response.setStatusLine(request.httpVersion, 400, "Bad Request");
- response.setHeader("Content-Type", "text/plain", false);
- response.bodyOutputStream.write(code, code.length);
-}
-
-/**
- * Represent a BSO on the server.
- *
- * A BSO is constructed from an ID, content, and a modified time.
- *
- * @param id
- * (string) ID of the BSO being created.
- * @param payload
- * (strong|object) Payload for the BSO. Should ideally be a string. If
- * an object is passed, it will be fed into JSON.stringify and that
- * output will be set as the payload.
- * @param modified
- * (number) Milliseconds since UNIX epoch that the BSO was last
- * modified. If not defined or null, the current time will be used.
- */
-this.ServerBSO = function ServerBSO(id, payload, modified) {
- if (!id) {
- throw new Error("No ID for ServerBSO!");
- }
-
- if (!id.match(/^[a-zA-Z0-9_-]{1,64}$/)) {
- throw new Error("BSO ID is invalid: " + id);
- }
-
- this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
-
- this.id = id;
- if (!payload) {
- return;
- }
-
- CommonUtils.ensureMillisecondsTimestamp(modified);
-
- if (typeof payload == "object") {
- payload = JSON.stringify(payload);
- }
-
- this.payload = payload;
- this.modified = modified || new_timestamp();
-};
-ServerBSO.prototype = {
- FIELDS: [
- "id",
- "modified",
- "payload",
- "ttl",
- "sortindex",
- ],
-
- toJSON: function toJSON() {
- let obj = {};
-
- for (let key of this.FIELDS) {
- if (this[key] !== undefined) {
- obj[key] = this[key];
- }
- }
-
- return obj;
- },
-
- delete: function delete_() {
- this.deleted = true;
-
- delete this.payload;
- delete this.modified;
- },
-
- /**
- * Handler for GET requests for this BSO.
- */
- getHandler: function getHandler(request, response) {
- let code = 200;
- let status = "OK";
- let body;
-
- function sendResponse() {
- response.setStatusLine(request.httpVersion, code, status);
- writeHttpBody(response, body);
- }
-
- if (request.hasHeader("x-if-modified-since")) {
- let headerModified = parseInt(request.getHeader("x-if-modified-since"),
- 10);
- CommonUtils.ensureMillisecondsTimestamp(headerModified);
-
- if (headerModified >= this.modified) {
- code = 304;
- status = "Not Modified";
-
- sendResponse();
- return;
- }
- } else if (request.hasHeader("x-if-unmodified-since")) {
- let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
- 10);
- let serverModified = this.modified;
-
- if (serverModified > requestModified) {
- code = 412;
- status = "Precondition Failed";
- sendResponse();
- return;
- }
- }
-
- if (!this.deleted) {
- body = JSON.stringify(this.toJSON());
- response.setHeader("Content-Type", "application/json", false);
- response.setHeader("X-Last-Modified", "" + this.modified, false);
- } else {
- code = 404;
- status = "Not Found";
- }
-
- sendResponse();
- },
-
- /**
- * Handler for PUT requests for this BSO.
- */
- putHandler: function putHandler(request, response) {
- if (request.hasHeader("Content-Type")) {
- let ct = request.getHeader("Content-Type");
- if (ct != "application/json") {
- throw HTTP_415;
- }
- }
-
- let input = CommonUtils.readBytesFromInputStream(request.bodyInputStream);
- let parsed;
- try {
- parsed = JSON.parse(input);
- } catch (ex) {
- sendMozSvcError(request, response, "8");
- return;
- }
-
- if (typeof(parsed) != "object") {
- sendMozSvcError(request, response, "8");
- return;
- }
-
- // Don't update if a conditional request fails preconditions.
- if (request.hasHeader("x-if-unmodified-since")) {
- let reqModified = parseInt(request.getHeader("x-if-unmodified-since"));
-
- if (reqModified < this.modified) {
- response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
- return;
- }
- }
-
- let code, status;
- if (this.payload) {
- code = 204;
- status = "No Content";
- } else {
- code = 201;
- status = "Created";
- }
-
- // Alert when we see unrecognized fields.
- for (let [key, value] of Object.entries(parsed)) {
- switch (key) {
- case "payload":
- if (typeof(value) != "string") {
- sendMozSvcError(request, response, "8");
- return;
- }
-
- this.payload = value;
- break;
-
- case "ttl":
- if (!isInteger(value)) {
- sendMozSvcError(request, response, "8");
- return;
- }
- this.ttl = parseInt(value, 10);
- break;
-
- case "sortindex":
- if (!isInteger(value) || value.length > 9) {
- sendMozSvcError(request, response, "8");
- return;
- }
- this.sortindex = parseInt(value, 10);
- break;
-
- case "id":
- break;
-
- default:
- this._log.warn("Unexpected field in BSO record: " + key);
- sendMozSvcError(request, response, "8");
- return;
- }
- }
-
- this.modified = request.timestamp;
- this.deleted = false;
- response.setHeader("X-Last-Modified", "" + this.modified, false);
-
- response.setStatusLine(request.httpVersion, code, status);
- },
-};
-
-/**
- * Represent a collection on the server.
- *
- * The '_bsos' attribute is a mapping of id -> ServerBSO objects.
- *
- * Note that if you want these records to be accessible individually,
- * you need to register their handlers with the server separately, or use a
- * containing HTTP server that will do so on your behalf.
- *
- * @param bsos
- * An object mapping BSO IDs to ServerBSOs.
- * @param acceptNew
- * If true, POSTs to this collection URI will result in new BSOs being
- * created and wired in on the fly.
- * @param timestamp
- * An optional timestamp value to initialize the modified time of the
- * collection. This should be in the format returned by new_timestamp().
- */
-this.StorageServerCollection =
- function StorageServerCollection(bsos, acceptNew, timestamp = new_timestamp()) {
- this._bsos = bsos || {};
- this.acceptNew = acceptNew || false;
-
- /*
- * Track modified timestamp.
- * We can't just use the timestamps of contained BSOs: an empty collection
- * has a modified time.
- */
- CommonUtils.ensureMillisecondsTimestamp(timestamp);
- this._timestamp = timestamp;
-
- this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
-};
-StorageServerCollection.prototype = {
- BATCH_MAX_COUNT: 100, // # of records.
- BATCH_MAX_SIZE: 1024 * 1024, // # bytes.
-
- _timestamp: null,
-
- get timestamp() {
- return this._timestamp;
- },
-
- set timestamp(timestamp) {
- CommonUtils.ensureMillisecondsTimestamp(timestamp);
- this._timestamp = timestamp;
- },
-
- get totalPayloadSize() {
- let size = 0;
- for (let bso of this.bsos()) {
- size += bso.payload.length;
- }
-
- return size;
- },
-
- /**
- * Convenience accessor for our BSO keys.
- * Excludes deleted items, of course.
- *
- * @param filter
- * A predicate function (applied to the ID and BSO) which dictates
- * whether to include the BSO's ID in the output.
- *
- * @return an array of IDs.
- */
- keys: function keys(filter) {
- let ids = [];
- for (let [id, bso] of Object.entries(this._bsos)) {
- if (!bso.deleted && (!filter || filter(id, bso))) {
- ids.push(id);
- }
- }
- return ids;
- },
-
- /**
- * Convenience method to get an array of BSOs.
- * Optionally provide a filter function.
- *
- * @param filter
- * A predicate function, applied to the BSO, which dictates whether to
- * include the BSO in the output.
- *
- * @return an array of ServerBSOs.
- */
- bsos: function bsos(filter) {
- let os = [];
- for (let bso of Object.values(this._bsos)) {
- if (!bso.deleted) {
- os.push(bso);
- }
- }
-
- if (!filter) {
- return os;
- }
-
- return os.filter(filter);
- },
-
- /**
- * Obtain a BSO by ID.
- */
- bso: function bso(id) {
- return this._bsos[id];
- },
-
- /**
- * Obtain the payload of a specific BSO.
- *
- * Raises if the specified BSO does not exist.
- */
- payload: function payload(id) {
- return this.bso(id).payload;
- },
-
- /**
- * Insert the provided BSO under its ID.
- *
- * @return the provided BSO.
- */
- insertBSO: function insertBSO(bso) {
- return this._bsos[bso.id] = bso;
- },
-
- /**
- * Insert the provided payload as part of a new ServerBSO with the provided
- * ID.
- *
- * @param id
- * The GUID for the BSO.
- * @param payload
- * The payload, as provided to the ServerBSO constructor.
- * @param modified
- * An optional modified time for the ServerBSO. If not specified, the
- * current time will be used.
- *
- * @return the inserted BSO.
- */
- insert: function insert(id, payload, modified) {
- return this.insertBSO(new ServerBSO(id, payload, modified));
- },
-
- /**
- * Removes an object entirely from the collection.
- *
- * @param id
- * (string) ID to remove.
- */
- remove: function remove(id) {
- delete this._bsos[id];
- },
-
- _inResultSet: function _inResultSet(bso, options) {
- if (!bso.payload) {
- return false;
- }
-
- if (options.ids) {
- if (options.ids.indexOf(bso.id) == -1) {
- return false;
- }
- }
-
- if (options.newer) {
- if (bso.modified <= options.newer) {
- return false;
- }
- }
-
- if (options.older) {
- if (bso.modified >= options.older) {
- return false;
- }
- }
-
- return true;
- },
-
- count: function count(options) {
- options = options || {};
- let c = 0;
- for (let bso of Object.values(this._bsos)) {
- if (bso.modified && this._inResultSet(bso, options)) {
- c++;
- }
- }
- return c;
- },
-
- get: function get(options) {
- let data = [];
- for (let id in this._bsos) {
- let bso = this._bsos[id];
- if (!bso.modified) {
- continue;
- }
-
- if (!this._inResultSet(bso, options)) {
- continue;
- }
-
- data.push(bso);
- }
-
- if (options.sort) {
- if (options.sort == "oldest") {
- data.sort(function sortOldest(a, b) {
- if (a.modified == b.modified) {
- return 0;
- }
-
- return a.modified < b.modified ? -1 : 1;
- });
- } else if (options.sort == "newest") {
- data.sort(function sortNewest(a, b) {
- if (a.modified == b.modified) {
- return 0;
- }
-
- return a.modified > b.modified ? -1 : 1;
- });
- } else if (options.sort == "index") {
- data.sort(function sortIndex(a, b) {
- if (a.sortindex == b.sortindex) {
- return 0;
- }
-
- if (a.sortindex !== undefined && b.sortindex == undefined) {
- return 1;
- }
-
- if (a.sortindex === undefined && b.sortindex !== undefined) {
- return -1;
- }
-
- return a.sortindex > b.sortindex ? -1 : 1;
- });
- }
- }
-
- if (options.limit) {
- data = data.slice(0, options.limit);
- }
-
- return data;
- },
-
- post: function post(input, timestamp) {
- let success = [];
- let failed = {};
- let count = 0;
- let size = 0;
-
- // This will count records where we have an existing ServerBSO
- // registered with us as successful and all other records as failed.
- for (let record of input) {
- count += 1;
- if (count > this.BATCH_MAX_COUNT) {
- failed[record.id] = "Max record count exceeded.";
- continue;
- }
-
- if (typeof(record.payload) != "string") {
- failed[record.id] = "Payload is not a string!";
- continue;
- }
-
- size += record.payload.length;
- if (size > this.BATCH_MAX_SIZE) {
- failed[record.id] = "Payload max size exceeded!";
- continue;
- }
-
- if (record.sortindex) {
- if (!isInteger(record.sortindex)) {
- failed[record.id] = "sortindex is not an integer.";
- continue;
- }
-
- if (record.sortindex.length > 9) {
- failed[record.id] = "sortindex is too long.";
- continue;
- }
- }
-
- if ("ttl" in record) {
- if (!isInteger(record.ttl)) {
- failed[record.id] = "ttl is not an integer.";
- continue;
- }
- }
-
- try {
- let bso = this.bso(record.id);
- if (!bso && this.acceptNew) {
- this._log.debug("Creating BSO " + JSON.stringify(record.id) +
- " on the fly.");
- bso = new ServerBSO(record.id);
- this.insertBSO(bso);
- }
- if (bso) {
- bso.payload = record.payload;
- bso.modified = timestamp;
- bso.deleted = false;
- success.push(record.id);
-
- if (record.sortindex) {
- bso.sortindex = parseInt(record.sortindex, 10);
- }
-
- } else {
- failed[record.id] = "no bso configured";
- }
- } catch (ex) {
- this._log.info("Exception when processing BSO", ex);
- failed[record.id] = "Exception when processing.";
- }
- }
- return {success, failed};
- },
-
- delete: function delete_(options) {
- options = options || {};
-
- // Protocol 2.0 only allows the "ids" query string argument.
- let keys = Object.keys(options).filter(function(k) {
- return k != "ids";
- });
- if (keys.length) {
- this._log.warn("Invalid query string parameter to collection delete: " +
- keys.join(", "));
- throw new Error("Malformed client request.");
- }
-
- if (options.ids && options.ids.length > this.BATCH_MAX_COUNT) {
- throw HTTP_400;
- }
-
- let deleted = [];
- for (let bso of Object.values(this._bsos)) {
- if (this._inResultSet(bso, options)) {
- this._log.debug("Deleting " + JSON.stringify(bso));
- deleted.push(bso.id);
- bso.delete();
- }
- }
- return deleted;
- },
-
- parseOptions: function parseOptions(request) {
- let options = {};
-
- for (let chunk of request.queryString.split("&")) {
- if (!chunk) {
- continue;
- }
- chunk = chunk.split("=");
- let key = decodeURIComponent(chunk[0]);
- if (chunk.length == 1) {
- options[key] = "";
- } else {
- options[key] = decodeURIComponent(chunk[1]);
- }
- }
-
- if (options.ids) {
- options.ids = options.ids.split(",");
- }
-
- if (options.newer) {
- if (!isInteger(options.newer)) {
- throw HTTP_400;
- }
-
- CommonUtils.ensureMillisecondsTimestamp(options.newer);
- options.newer = parseInt(options.newer, 10);
- }
-
- if (options.older) {
- if (!isInteger(options.older)) {
- throw HTTP_400;
- }
-
- CommonUtils.ensureMillisecondsTimestamp(options.older);
- options.older = parseInt(options.older, 10);
- }
-
- if (options.limit) {
- if (!isInteger(options.limit)) {
- throw HTTP_400;
- }
-
- options.limit = parseInt(options.limit, 10);
- }
-
- return options;
- },
-
- getHandler: function getHandler(request, response) {
- let options = this.parseOptions(request);
- let data = this.get(options);
-
- if (request.hasHeader("x-if-modified-since")) {
- let requestModified = parseInt(request.getHeader("x-if-modified-since"),
- 10);
- let newestBSO = 0;
- for (let bso of data) {
- if (bso.modified > newestBSO) {
- newestBSO = bso.modified;
- }
- }
-
- if (requestModified >= newestBSO) {
- response.setHeader("X-Last-Modified", "" + newestBSO);
- response.setStatusLine(request.httpVersion, 304, "Not Modified");
- return;
- }
- } else if (request.hasHeader("x-if-unmodified-since")) {
- let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
- 10);
- let serverModified = this.timestamp;
-
- if (serverModified > requestModified) {
- response.setHeader("X-Last-Modified", "" + serverModified);
- response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
- return;
- }
- }
-
- if (options.full) {
- data = data.map(function map(bso) {
- return bso.toJSON();
- });
- } else {
- data = data.map(function map(bso) {
- return bso.id;
- });
- }
-
- // application/json is default media type.
- let newlines = false;
- if (request.hasHeader("accept")) {
- let accept = request.getHeader("accept");
- if (accept == "application/newlines") {
- newlines = true;
- } else if (accept != "application/json") {
- throw HTTP_406;
- }
- }
-
- let body;
- if (newlines) {
- response.setHeader("Content-Type", "application/newlines", false);
- let normalized = data.map(function map(d) {
- return JSON.stringify(d);
- });
-
- body = normalized.join("\n") + "\n";
- } else {
- response.setHeader("Content-Type", "application/json", false);
- body = JSON.stringify({items: data});
- }
-
- this._log.info("Records: " + data.length);
- response.setHeader("X-Num-Records", "" + data.length, false);
- response.setHeader("X-Last-Modified", "" + this.timestamp, false);
- response.setStatusLine(request.httpVersion, 200, "OK");
- response.bodyOutputStream.write(body, body.length);
- },
-
- postHandler: function postHandler(request, response) {
- if (!request.hasHeader("content-type")) {
- this._log.info("No Content-Type request header!");
- throw HTTP_400;
- }
-
- let inputStream = request.bodyInputStream;
- let inputBody = CommonUtils.readBytesFromInputStream(inputStream);
- let input = [];
-
- let inputMediaType = request.getHeader("content-type");
- if (inputMediaType == "application/json") {
- try {
- input = JSON.parse(inputBody);
- } catch (ex) {
- this._log.info("JSON parse error on input body!");
- throw HTTP_400;
- }
-
- if (!Array.isArray(input)) {
- this._log.info("Input JSON type not an array!");
- sendMozSvcError(request, response, "8");
- return;
- }
- } else if (inputMediaType == "application/newlines") {
- for (let line of inputBody.split("\n")) {
- let record;
- try {
- record = JSON.parse(line);
- } catch (ex) {
- this._log.info("JSON parse error on line!");
- sendMozSvcError(request, response, "8");
- return;
- }
-
- input.push(record);
- }
- } else {
- this._log.info("Unknown media type: " + inputMediaType);
- throw HTTP_415;
- }
-
- if (this._ensureUnmodifiedSince(request, response)) {
- return;
- }
-
- let res = this.post(input, request.timestamp);
- let body = JSON.stringify(res);
- response.setHeader("Content-Type", "application/json", false);
- this.timestamp = request.timestamp;
- response.setHeader("X-Last-Modified", "" + this.timestamp, false);
-
- response.setStatusLine(request.httpVersion, "200", "OK");
- response.bodyOutputStream.write(body, body.length);
- },
-
- deleteHandler: function deleteHandler(request, response) {
- this._log.debug("Invoking StorageServerCollection.DELETE.");
-
- let options = this.parseOptions(request);
-
- if (this._ensureUnmodifiedSince(request, response)) {
- return;
- }
-
- let deleted = this.delete(options);
- response.deleted = deleted;
- this.timestamp = request.timestamp;
-
- response.setStatusLine(request.httpVersion, 204, "No Content");
- },
-
- handler: function handler() {
- let self = this;
-
- return function(request, response) {
- switch (request.method) {
- case "GET":
- self.getHandler(request, response);
- return;
-
- case "POST":
- self.postHandler(request, response);
- return;
-
- case "DELETE":
- self.deleteHandler(request, response);
- return;
-
- }
-
- request.setHeader("Allow", "GET,POST,DELETE");
- response.setStatusLine(request.httpVersion, 405, "Method Not Allowed");
- };
- },
-
- _ensureUnmodifiedSince: function _ensureUnmodifiedSince(request, response) {
- if (!request.hasHeader("x-if-unmodified-since")) {
- return false;
- }
-
- let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
- 10);
- let serverModified = this.timestamp;
-
- this._log.debug("Request modified time: " + requestModified +
- "; Server modified time: " + serverModified);
- if (serverModified <= requestModified) {
- return false;
- }
-
- this._log.info("Conditional request rejected because client time older " +
- "than collection timestamp.");
- response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
- return true;
- },
-};
-
-
-// ===========================================================================//
-// httpd.js-based Storage server. //
-// ===========================================================================//
-
-/**
- * In general, the preferred way of using StorageServer is to directly
- * introspect it. Callbacks are available for operations which are hard to
- * verify through introspection, such as deletions.
- *
- * One of the goals of this server is to provide enough hooks for test code to
- * find out what it needs without monkeypatching. Use this object as your
- * prototype, and override as appropriate.
- */
-this.StorageServerCallback = {
- onCollectionDeleted: function onCollectionDeleted(user, collection) {},
- onItemDeleted: function onItemDeleted(user, collection, bsoID) {},
-
- /**
- * Called at the top of every request.
- *
- * Allows the test to inspect the request. Hooks should be careful not to
- * modify or change state of the request or they may impact future processing.
- */
- onRequest: function onRequest(request) {},
-};
-
-/**
- * Construct a new test Storage server. Takes a callback object (e.g.,
- * StorageServerCallback) as input.
- */
-this.StorageServer = function StorageServer(callback) {
- this.callback = callback || {__proto__: StorageServerCallback};
- this.server = new HttpServer();
- this.started = false;
- this.users = {};
- this.requestCount = 0;
- this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
-
- // Install our own default handler. This allows us to mess around with the
- // whole URL space.
- let handler = this.server._handler;
- handler._handleDefault = this.handleDefault.bind(this, handler);
-};
-StorageServer.prototype = {
- DEFAULT_QUOTA: 1024 * 1024, // # bytes.
-
- server: null, // HttpServer.
- users: null, // Map of username => {collections, password}.
-
- /**
- * If true, the server will allow any arbitrary user to be used.
- *
- * No authentication will be performed. Whatever user is detected from the
- * URL or auth headers will be created (if needed) and used.
- */
- allowAllUsers: false,
-
- /**
- * Start the StorageServer's underlying HTTP server.
- *
- * @param port
- * The numeric port on which to start. A falsy value implies to
- * select any available port.
- * @param cb
- * A callback function (of no arguments) which is invoked after
- * startup.
- */
- start: function start(port, cb) {
- if (this.started) {
- this._log.warn("Warning: server already started on " + this.port);
- return;
- }
- if (!port) {
- port = -1;
- }
- this.port = port;
-
- try {
- this.server.start(this.port);
- this.port = this.server.identity.primaryPort;
- this.started = true;
- if (cb) {
- cb();
- }
- } catch (ex) {
- this._log.error("==========================================");
- this._log.error("Got exception starting Storage HTTP server on port " + this.port);
- this._log.error("Error: " + Log.exceptionStr(ex));
- this._log.error("Is there a process already listening on port " + this.port + "?");
- this._log.error("==========================================");
- throw ex;
- }
- },
-
- /**
- * Start the server synchronously.
- *
- * @param port
- * The numeric port on which to start. The default is to choose
- * any available port.
- */
- startSynchronous: function startSynchronous(port = -1) {
- let cb = Async.makeSpinningCallback();
- this.start(port, cb);
- cb.wait();
- },
-
- /**
- * Stop the StorageServer's HTTP server.
- *
- * @param cb
- * A callback function. Invoked after the server has been stopped.
- *
- */
- stop: function stop(cb) {
- if (!this.started) {
- this._log.warn("StorageServer: Warning: server not running. Can't stop " +
- "me now!");
- return;
- }
-
- this.server.stop(cb);
- this.started = false;
- },
-
- serverTime: function serverTime() {
- return new_timestamp();
- },
-
- /**
- * Create a new user, complete with an empty set of collections.
- *
- * @param username
- * The username to use. An Error will be thrown if a user by that name
- * already exists.
- * @param password
- * A password string.
- *
- * @return a user object, as would be returned by server.user(username).
- */
- registerUser: function registerUser(username, password) {
- if (username in this.users) {
- throw new Error("User already exists.");
- }
-
- if (!isFinite(parseInt(username))) {
- throw new Error("Usernames must be numeric: " + username);
- }
-
- this._log.info("Registering new user with server: " + username);
- this.users[username] = {
- password,
- collections: {},
- quota: this.DEFAULT_QUOTA,
- };
- return this.user(username);
- },
-
- userExists: function userExists(username) {
- return username in this.users;
- },
-
- getCollection: function getCollection(username, collection) {
- return this.users[username].collections[collection];
- },
-
- _insertCollection: function _insertCollection(collections, collection, bsos) {
- let coll = new StorageServerCollection(bsos, true);
- coll.collectionHandler = coll.handler();
- collections[collection] = coll;
- return coll;
- },
-
- createCollection: function createCollection(username, collection, bsos) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
- let collections = this.users[username].collections;
- if (collection in collections) {
- throw new Error("Collection already exists.");
- }
- return this._insertCollection(collections, collection, bsos);
- },
-
- deleteCollection: function deleteCollection(username, collection) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
- delete this.users[username].collections[collection];
- },
-
- /**
- * Accept a map like the following:
- * {
- * meta: {global: {version: 1, ...}},
- * crypto: {"keys": {}, foo: {bar: 2}},
- * bookmarks: {}
- * }
- * to cause collections and BSOs to be created.
- * If a collection already exists, no error is raised.
- * If a BSO already exists, it will be updated to the new contents.
- */
- createContents: function createContents(username, collections) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
- let userCollections = this.users[username].collections;
- for (let [id, contents] of Object.entries(collections)) {
- let coll = userCollections[id] ||
- this._insertCollection(userCollections, id);
- for (let [bsoID, payload] of Object.entries(contents)) {
- coll.insert(bsoID, payload);
- }
- }
- },
-
- /**
- * Insert a BSO in an existing collection.
- */
- insertBSO: function insertBSO(username, collection, bso) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
- let userCollections = this.users[username].collections;
- if (!(collection in userCollections)) {
- throw new Error("Unknown collection.");
- }
- userCollections[collection].insertBSO(bso);
- return bso;
- },
-
- /**
- * Delete all of the collections for the named user.
- *
- * @param username
- * The name of the affected user.
- */
- deleteCollections: function deleteCollections(username) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
- let userCollections = this.users[username].collections;
- for (let name in userCollections) {
- let coll = userCollections[name];
- this._log.trace("Bulk deleting " + name + " for " + username + "...");
- coll.delete({});
- }
- this.users[username].collections = {};
- },
-
- getQuota: function getQuota(username) {
- if (!(username in this.users)) {
- throw new Error("Unknown user.");
- }
-
- return this.users[username].quota;
- },
-
- /**
- * Obtain the newest timestamp of all collections for a user.
- */
- newestCollectionTimestamp: function newestCollectionTimestamp(username) {
- let collections = this.users[username].collections;
- let newest = 0;
- for (let name in collections) {
- let collection = collections[name];
- if (collection.timestamp > newest) {
- newest = collection.timestamp;
- }
- }
-
- return newest;
- },
-
- /**
- * Compute the object that is returned for an info/collections request.
- */
- infoCollections: function infoCollections(username) {
- let responseObject = {};
- let colls = this.users[username].collections;
- for (let coll in colls) {
- responseObject[coll] = colls[coll].timestamp;
- }
- this._log.trace("StorageServer: info/collections returning " +
- JSON.stringify(responseObject));
- return responseObject;
- },
-
- infoCounts: function infoCounts(username) {
- let data = {};
- let collections = this.users[username].collections;
- for (let [k, v] of Object.entries(collections)) {
- let count = v.count();
- if (!count) {
- continue;
- }
-
- data[k] = count;
- }
-
- return data;
- },
-
- infoUsage: function infoUsage(username) {
- let data = {};
- let collections = this.users[username].collections;
- for (let [k, v] of Object.entries(collections)) {
- data[k] = v.totalPayloadSize;
- }
-
- return data;
- },
-
- infoQuota: function infoQuota(username) {
- let total = 0;
- let usage = this.infoUsage(username);
- for (let key in usage) {
- let value = usage[key];
- total += value;
- }
-
- return {
- quota: this.getQuota(username),
- usage: total
- };
- },
-
- /**
- * Simple accessor to allow collective binding and abbreviation of a bunch of
- * methods. Yay!
- * Use like this:
- *
- * let u = server.user("john");
- * u.collection("bookmarks").bso("abcdefg").payload; // Etc.
- *
- * @return a proxy for the user data stored in this server.
- */
- user: function user(username) {
- let collection = this.getCollection.bind(this, username);
- let createCollection = this.createCollection.bind(this, username);
- let createContents = this.createContents.bind(this, username);
- let modified = function(collectionName) {
- return collection(collectionName).timestamp;
- };
- let deleteCollections = this.deleteCollections.bind(this, username);
- let quota = this.getQuota.bind(this, username);
- return {
- collection,
- createCollection,
- createContents,
- deleteCollections,
- modified,
- quota,
- };
- },
-
- _pruneExpired: function _pruneExpired() {
- let now = Date.now();
-
- for (let username in this.users) {
- let user = this.users[username];
- for (let name in user.collections) {
- let collection = user.collections[name];
- for (let bso of collection.bsos()) {
- // ttl === 0 is a special case, so we can't simply !ttl.
- if (typeof(bso.ttl) != "number") {
- continue;
- }
-
- let ttlDate = bso.modified + (bso.ttl * 1000);
- if (ttlDate < now) {
- this._log.info("Deleting BSO because TTL expired: " + bso.id);
- bso.delete();
- }
- }
- }
- }
- },
-
- /*
- * Regular expressions for splitting up Storage request paths.
- * Storage URLs are of the form:
- * /$apipath/$version/$userid/$further
- * where $further is usually:
- * storage/$collection/$bso
- * or
- * storage/$collection
- * or
- * info/$op
- *
- * We assume for the sake of simplicity that $apipath is empty.
- *
- * N.B., we don't follow any kind of username spec here, because as far as I
- * can tell there isn't one. See Bug 689671. Instead we follow the Python
- * server code.
- *
- * Path: [all, version, first, rest]
- * Storage: [all, collection?, id?]
- */
- pathRE: /^\/([0-9]+(?:\.[0-9]+)?)(?:\/([0-9]+)\/([^\/]+)(?:\/(.+))?)?$/,
- storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/,
-
- defaultHeaders: {},
-
- /**
- * HTTP response utility.
- */
- respond: function respond(req, resp, code, status, body, headers, timestamp) {
- this._log.info("Response: " + code + " " + status);
- resp.setStatusLine(req.httpVersion, code, status);
- if (!headers) {
- headers = this.defaultHeaders;
- }
- for (let header in headers) {
- let value = headers[header];
- resp.setHeader(header, value, false);
- }
-
- if (timestamp) {
- resp.setHeader("X-Timestamp", "" + timestamp, false);
- }
-
- if (body) {
- resp.bodyOutputStream.write(body, body.length);
- }
- },
-
- /**
- * This is invoked by the HttpServer. `this` is bound to the StorageServer;
- * `handler` is the HttpServer's handler.
- *
- * TODO: need to use the correct Storage API response codes and errors here.
- */
- handleDefault: function handleDefault(handler, req, resp) {
- this.requestCount++;
- let timestamp = new_timestamp();
- try {
- this._handleDefault(handler, req, resp, timestamp);
- } catch (e) {
- if (e instanceof HttpError) {
- this.respond(req, resp, e.code, e.description, "", {}, timestamp);
- } else {
- this._log.warn("StorageServer: handleDefault caught an error", e);
- throw e;
- }
- }
- },
-
- _handleDefault: function _handleDefault(handler, req, resp, timestamp) {
- let path = req.path;
- if (req.queryString.length) {
- path += "?" + req.queryString;
- }
-
- this._log.debug("StorageServer: Handling request: " + req.method + " " +
- path);
-
- if (this.callback.onRequest) {
- this.callback.onRequest(req);
- }
-
- // Prune expired records for all users at top of request. This is the
- // easiest way to process TTLs since all requests go through here.
- this._pruneExpired();
-
- req.timestamp = timestamp;
- resp.setHeader("X-Timestamp", "" + timestamp, false);
-
- let parts = this.pathRE.exec(req.path);
- if (!parts) {
- this._log.debug("StorageServer: Unexpected request: bad URL " + req.path);
- throw HTTP_404;
- }
-
- let [, version, userPath, first, rest] = parts;
- if (version != STORAGE_API_VERSION) {
- this._log.debug("StorageServer: Unknown version.");
- throw HTTP_404;
- }
-
- let username;
-
- // By default, the server requires users to be authenticated. When a
- // request arrives, the user must have been previously configured and
- // the request must have authentication. In "allow all users" mode, we
- // take the username from the URL, create the user on the fly, and don't
- // perform any authentication.
- if (!this.allowAllUsers) {
- // Enforce authentication.
- if (!req.hasHeader("authorization")) {
- this.respond(req, resp, 401, "Authorization Required", "{}", {
- "WWW-Authenticate": 'Basic realm="secret"'
- });
- return;
- }
-
- let ensureUserExists = username => {
- if (this.userExists(username)) {
- return;
- }
-
- this._log.info("StorageServer: Unknown user: " + username);
- throw HTTP_401;
- };
-
- let auth = req.getHeader("authorization");
- this._log.debug("Authorization: " + auth);
-
- if (auth.indexOf("Basic ") == 0) {
- let decoded = CommonUtils.safeAtoB(auth.substr(6));
- this._log.debug("Decoded Basic Auth: " + decoded);
- let [user, password] = decoded.split(":", 2);
-
- if (!password) {
- this._log.debug("Malformed HTTP Basic Authorization header: " + auth);
- throw HTTP_400;
- }
-
- this._log.debug("Got HTTP Basic auth for user: " + user);
- ensureUserExists(user);
- username = user;
-
- if (this.users[user].password != password) {
- this._log.debug("StorageServer: Provided password is not correct.");
- throw HTTP_401;
- }
- // TODO support token auth.
- } else {
- this._log.debug("Unsupported HTTP authorization type: " + auth);
- throw HTTP_500;
- }
- // All users mode.
- } else {
- // Auto create user with dummy password.
- if (!this.userExists(userPath)) {
- this.registerUser(userPath, "DUMMY-PASSWORD-*&%#");
- }
-
- username = userPath;
- }
-
- // Hand off to the appropriate handler for this path component.
- if (first in this.toplevelHandlers) {
- let handler = this.toplevelHandlers[first];
- try {
- handler.call(this, handler, req, resp, version, username, rest);
- return;
- } catch (ex) {
- this._log.warn("Got exception during request", ex);
- throw ex;
- }
- }
- this._log.debug("StorageServer: Unknown top-level " + first);
- throw HTTP_404;
- },
-
- /**
- * Collection of the handler methods we use for top-level path components.
- */
- toplevelHandlers: {
- "storage": function handleStorage(handler, req, resp, version, username,
- rest) {
- let respond = this.respond.bind(this, req, resp);
- if (!rest || !rest.length) {
- this._log.debug("StorageServer: top-level storage " +
- req.method + " request.");
-
- if (req.method != "DELETE") {
- respond(405, "Method Not Allowed", null, {"Allow": "DELETE"});
- return;
- }
-
- this.user(username).deleteCollections();
-
- respond(204, "No Content");
- return;
- }
-
- let match = this.storageRE.exec(rest);
- if (!match) {
- this._log.warn("StorageServer: Unknown storage operation " + rest);
- throw HTTP_404;
- }
- let [, collection, bsoID] = match;
- let coll = this.getCollection(username, collection);
- let collectionExisted = !!coll;
-
- switch (req.method) {
- case "GET":
- // Tried to GET on a collection that doesn't exist.
- if (!coll) {
- respond(404, "Not Found");
- return;
- }
-
- // No BSO URL parameter goes to collection handler.
- if (!bsoID) {
- coll.collectionHandler(req, resp);
- return;
- }
-
- // Handle non-existent BSO.
- let bso = coll.bso(bsoID);
- if (!bso) {
- respond(404, "Not Found");
- return;
- }
-
- // Proxy to BSO handler.
- bso.getHandler(req, resp);
- return;
-
- case "DELETE":
- // Collection doesn't exist.
- if (!coll) {
- respond(404, "Not Found");
- return;
- }
-
- // Deleting a specific BSO.
- if (bsoID) {
- let bso = coll.bso(bsoID);
-
- // BSO does not exist on the server. Nothing to do.
- if (!bso) {
- respond(404, "Not Found");
- return;
- }
-
- if (req.hasHeader("x-if-unmodified-since")) {
- let modified = parseInt(req.getHeader("x-if-unmodified-since"));
- CommonUtils.ensureMillisecondsTimestamp(modified);
-
- if (bso.modified > modified) {
- respond(412, "Precondition Failed");
- return;
- }
- }
-
- bso.delete();
- coll.timestamp = req.timestamp;
- this.callback.onItemDeleted(username, collection, bsoID);
- respond(204, "No Content");
- return;
- }
-
- // Proxy to collection handler.
- coll.collectionHandler(req, resp);
-
- // Spot if this is a DELETE for some IDs, and don't blow away the
- // whole collection!
- //
- // We already handled deleting the BSOs by invoking the deleted
- // collection's handler. However, in the case of
- //
- // DELETE storage/foobar
- //
- // we also need to remove foobar from the collections map. This
- // clause tries to differentiate the above request from
- //
- // DELETE storage/foobar?ids=foo,baz
- //
- // and do the right thing.
- // TODO: less hacky method.
- if (-1 == req.queryString.indexOf("ids=")) {
- // When you delete the entire collection, we drop it.
- this._log.debug("Deleting entire collection.");
- delete this.users[username].collections[collection];
- this.callback.onCollectionDeleted(username, collection);
- }
-
- // Notify of item deletion.
- let deleted = resp.deleted || [];
- for (let i = 0; i < deleted.length; ++i) {
- this.callback.onItemDeleted(username, collection, deleted[i]);
- }
- return;
-
- case "POST":
- case "PUT":
- // Auto-create collection if it doesn't exist.
- if (!coll) {
- coll = this.createCollection(username, collection);
- }
-
- try {
- if (bsoID) {
- let bso = coll.bso(bsoID);
- if (!bso) {
- this._log.trace("StorageServer: creating BSO " + collection +
- "/" + bsoID);
- try {
- bso = coll.insert(bsoID);
- } catch (ex) {
- sendMozSvcError(req, resp, "8");
- return;
- }
- }
-
- bso.putHandler(req, resp);
-
- coll.timestamp = req.timestamp;
- return;
- }
-
- coll.collectionHandler(req, resp);
- return;
- } catch (ex) {
- if (ex instanceof HttpError) {
- if (!collectionExisted) {
- this.deleteCollection(username, collection);
- }
- }
-
- throw ex;
- }
-
- default:
- throw new Error("Request method " + req.method + " not implemented.");
- }
- },
-
- "info": function handleInfo(handler, req, resp, version, username, rest) {
- switch (rest) {
- case "collections":
- this.handleInfoCollections(req, resp, username);
- return;
-
- case "collection_counts":
- this.handleInfoCounts(req, resp, username);
- return;
-
- case "collection_usage":
- this.handleInfoUsage(req, resp, username);
- return;
-
- case "quota":
- this.handleInfoQuota(req, resp, username);
- return;
-
- default:
- this._log.warn("StorageServer: Unknown info operation " + rest);
- throw HTTP_404;
- }
- }
- },
-
- handleInfoConditional: function handleInfoConditional(request, response,
- user) {
- if (!request.hasHeader("x-if-modified-since")) {
- return false;
- }
-
- let requestModified = request.getHeader("x-if-modified-since");
- requestModified = parseInt(requestModified, 10);
-
- let serverModified = this.newestCollectionTimestamp(user);
-
- this._log.info("Server mtime: " + serverModified + "; Client modified: " +
- requestModified);
- if (serverModified > requestModified) {
- return false;
- }
-
- this.respond(request, response, 304, "Not Modified", null, {
- "X-Last-Modified": "" + serverModified
- });
-
- return true;
- },
-
- handleInfoCollections: function handleInfoCollections(request, response,
- user) {
- if (this.handleInfoConditional(request, response, user)) {
- return;
- }
-
- let info = this.infoCollections(user);
- let body = JSON.stringify(info);
- this.respond(request, response, 200, "OK", body, {
- "Content-Type": "application/json",
- "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
- });
- },
-
- handleInfoCounts: function handleInfoCounts(request, response, user) {
- if (this.handleInfoConditional(request, response, user)) {
- return;
- }
-
- let counts = this.infoCounts(user);
- let body = JSON.stringify(counts);
-
- this.respond(request, response, 200, "OK", body, {
- "Content-Type": "application/json",
- "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
- });
- },
-
- handleInfoUsage: function handleInfoUsage(request, response, user) {
- if (this.handleInfoConditional(request, response, user)) {
- return;
- }
-
- let body = JSON.stringify(this.infoUsage(user));
- this.respond(request, response, 200, "OK", body, {
- "Content-Type": "application/json",
- "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
- });
- },
-
- handleInfoQuota: function handleInfoQuota(request, response, user) {
- if (this.handleInfoConditional(request, response, user)) {
- return;
- }
-
- let body = JSON.stringify(this.infoQuota(user));
- this.respond(request, response, 200, "OK", body, {
- "Content-Type": "application/json",
- "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
- });
- },
-};
-
-/**
- * Helper to create a storage server for a set of users.
- *
- * Each user is specified by a map of username to password.
- */
-this.storageServerForUsers =
- function storageServerForUsers(users, contents, callback) {
- let server = new StorageServer(callback);
- for (let [user, pass] of Object.entries(users)) {
- server.registerUser(user, pass);
- server.createContents(user, contents);
- }
- server.start();
- return server;
-};
--- a/services/common/moz.build
+++ b/services/common/moz.build
@@ -29,20 +29,16 @@ EXTRA_JS_MODULES['services-common'] += [
if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
EXTRA_JS_MODULES['services-common'] += [
'hawkclient.js',
'hawkrequest.js',
'tokenserverclient.js',
]
- TESTING_JS_MODULES.services.common += [
- 'modules-testing/storageserver.js',
- ]
-
TESTING_JS_MODULES.services.common += [
'modules-testing/logging.js',
]
JS_PREFERENCE_FILES += [
'services-common.js',
]
deleted file mode 100644
--- a/services/common/tests/mach_commands.py
+++ /dev/null
@@ -1,111 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, unicode_literals
-
-import mozpack.path as mozpath
-
-from mozbuild.base import (
- MachCommandBase,
-)
-
-from mach.decorators import (
- CommandArgument,
- CommandProvider,
- Command,
-)
-
-from mach.registrar import (
- Registrar
-)
-
-from shutil import rmtree
-from subprocess import Popen
-from sys import argv
-from sys import exit
-from tempfile import mkdtemp
-
-
-
-DEFAULT_PORT = 8080
-DEFAULT_HOSTNAME = 'localhost'
-
-SRCDIR = mozpath.abspath(mozpath.dirname(__file__))
-
-STORAGE_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_storage_server.js')
-
-def SyncStorageCommand(func):
- """Decorator that adds shared command arguments to services commands."""
-
- port = CommandArgument('--port', metavar='PORT', type=int,
- default=DEFAULT_PORT, help='Port to run server on.')
- func = port(func)
-
- address = CommandArgument('--address', metavar='ADDRESS',
- default=DEFAULT_HOSTNAME,
- help='Hostname to bind server to.')
- func = address(func)
-
- return func
-
-Registrar.register_category(name='services',
- title='Services utilities',
- description='Commands for services development.')
-
-@CommandProvider
-class SyncTestCommands(MachCommandBase):
- def __init__(self, context):
- MachCommandBase.__init__(self, context)
-
- def run_server(self, js_file, hostname, port):
- topsrcdir = self.topsrcdir
- topobjdir = self.topobjdir
-
- unit_test_dir = mozpath.join(SRCDIR, 'unit')
-
- head_paths = [
- 'head_global.js',
- 'head_helpers.js',
- 'head_http.js',
- ]
-
- head_paths = ['"%s"' % mozpath.join(unit_test_dir, path) for path in head_paths]
-
- args = [
- '%s/run-mozilla.sh' % self.bindir,
- '%s/xpcshell' % self.bindir,
- '-g', self.bindir,
- '-a', self.bindir,
- '-r', '%s/components/httpd.manifest' % self.bindir,
- '-m',
- '-s',
- '-e', 'const _TESTING_MODULES_DIR = "%s/_tests/modules";' % topobjdir,
- '-f', '%s/testing/xpcshell/head.js' % topsrcdir,
- '-e', 'const _SERVER_ADDR = "%s";' % hostname,
- '-e', 'const SERVER_PORT = "%s";' % port,
- '-e', 'const INCLUDE_FILES = [%s];' % ', '.join(head_paths),
- '-e', '_register_protocol_handlers();',
- '-e', 'for (let name of INCLUDE_FILES) load(name);',
- '-e', '_fakeIdleService.activate();',
- '-f', js_file
- ]
-
- profile_dir = mkdtemp()
- print 'Created profile directory: %s' % profile_dir
-
- try:
- env = {'XPCSHELL_TEST_PROFILE_DIR': profile_dir}
- proc = Popen(args, env=env)
-
- return proc.wait()
-
- finally:
- print 'Removing profile directory %s' % profile_dir
- rmtree(profile_dir)
-
- @Command('storage-server', category='services',
- description='Run a storage server.')
- @SyncStorageCommand
- def run_storage_server(self, port=DEFAULT_PORT, address=DEFAULT_HOSTNAME):
- exit(self.run_server(STORAGE_SERVER_SCRIPT, address, port))
deleted file mode 100644
--- a/services/common/tests/run_storage_server.js
+++ /dev/null
@@ -1,29 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/**
- * This file runs a Storage Service server.
- *
- * It is meant to be executed with an xpcshell.
- *
- * The Makefile in this directory contains a target to run it:
- *
- * $ make storage-server
- */
-
-// Disable eslint no-undef rule for this file, as it is simple and is complicated
-// to check all the imports.
-/* eslint no-undef:off */
-
-Cu.import("resource://testing-common/services/common/storageserver.js");
-
-initTestLogging();
-
-var server = new StorageServer();
-server.allowAllUsers = true;
-server.startSynchronous(SERVER_PORT);
-_("Storage server started on port " + SERVER_PORT);
-
-// Launch the thread manager.
-_do_main();
--- a/services/common/tests/unit/test_load_modules.js
+++ b/services/common/tests/unit/test_load_modules.js
@@ -15,20 +15,16 @@ const non_android_modules = [
"tokenserverclient.js",
];
const TEST_BASE = "resource://testing-common/services/common/";
const shared_test_modules = [
"logging.js",
];
-const non_android_test_modules = [
- "storageserver.js",
-];
-
function expectImportsToSucceed(mm, base = MODULE_BASE) {
for (let m of mm) {
let resource = base + m;
let succeeded = false;
try {
Components.utils.import(resource, {});
succeeded = true;
} catch (e) {}
@@ -55,14 +51,12 @@ function expectImportsToFail(mm, base =
}
function run_test() {
expectImportsToSucceed(shared_modules);
expectImportsToSucceed(shared_test_modules, TEST_BASE);
if (AppConstants.platform != "android") {
expectImportsToSucceed(non_android_modules);
- expectImportsToSucceed(non_android_test_modules, TEST_BASE);
} else {
expectImportsToFail(non_android_modules);
- expectImportsToFail(non_android_test_modules, TEST_BASE);
}
}
deleted file mode 100644
--- a/services/common/tests/unit/test_storage_server.js
+++ /dev/null
@@ -1,692 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- http://creativecommons.org/publicdomain/zero/1.0/ */
-
-Cu.import("resource://services-common/async.js");
-Cu.import("resource://services-common/rest.js");
-Cu.import("resource://services-common/utils.js");
-Cu.import("resource://testing-common/services/common/storageserver.js");
-
-const DEFAULT_USER = "123";
-const DEFAULT_PASSWORD = "password";
-
-/**
- * Helper function to prepare a RESTRequest against the server.
- */
-function localRequest(server, path, user = DEFAULT_USER, password = DEFAULT_PASSWORD) {
- _("localRequest: " + path);
- let identity = server.server.identity;
- let url = identity.primaryScheme + "://" + identity.primaryHost + ":" +
- identity.primaryPort + path;
- _("url: " + url);
- let req = new RESTRequest(url);
-
- let header = basic_auth_header(user, password);
- req.setHeader("Authorization", header);
- req.setHeader("Accept", "application/json");
-
- return req;
-}
-
-/**
- * Helper function to validate an HTTP response from the server.
- */
-function validateResponse(response) {
- do_check_true("x-timestamp" in response.headers);
-
- if ("content-length" in response.headers) {
- let cl = parseInt(response.headers["content-length"]);
-
- if (cl != 0) {
- do_check_true("content-type" in response.headers);
- do_check_eq("application/json", response.headers["content-type"]);
- }
- }
-
- if (response.status == 204 || response.status == 304) {
- do_check_false("content-type" in response.headers);
-
- if ("content-length" in response.headers) {
- do_check_eq(response.headers["content-length"], "0");
- }
- }
-
- if (response.status == 405) {
- do_check_true("allow" in response.headers);
- }
-}
-
-/**
- * Helper function to synchronously wait for a response and validate it.
- */
-function waitAndValidateResponse(cb, request) {
- let error = cb.wait();
-
- if (!error) {
- validateResponse(request.response);
- }
-
- return error;
-}
-
-/**
- * Helper function to synchronously perform a GET request.
- *
- * @return Error instance or null if no error.
- */
-function doGetRequest(request) {
- let cb = Async.makeSpinningCallback();
- request.get(cb);
-
- return waitAndValidateResponse(cb, request);
-}
-
-/**
- * Helper function to synchronously perform a PUT request.
- *
- * @return Error instance or null if no error.
- */
-function doPutRequest(request, data) {
- let cb = Async.makeSpinningCallback();
- request.put(data, cb);
-
- return waitAndValidateResponse(cb, request);
-}
-
-/**
- * Helper function to synchronously perform a DELETE request.
- *
- * @return Error or null if no error was encountered.
- */
-function doDeleteRequest(request) {
- let cb = Async.makeSpinningCallback();
- request.delete(cb);
-
- return waitAndValidateResponse(cb, request);
-}
-
-function run_test() {
- Log.repository.getLogger("Services.Common.Test.StorageServer").level =
- Log.Level.Trace;
- initTestLogging();
-
- run_next_test();
-}
-
-add_test(function test_creation() {
- _("Ensure a simple server can be created.");
-
- // Explicit callback for this one.
- let server = new StorageServer({
- __proto__: StorageServerCallback,
- });
- do_check_true(!!server);
-
- server.start(-1, function() {
- _("Started on " + server.port);
- server.stop(run_next_test);
- });
-});
-
-add_test(function test_synchronous_start() {
- _("Ensure starting using startSynchronous works.");
-
- let server = new StorageServer();
- server.startSynchronous();
- server.stop(run_next_test);
-});
-
-add_test(function test_url_parsing() {
- _("Ensure server parses URLs properly.");
-
- let server = new StorageServer();
-
- // Check that we can parse a BSO URI.
- let parts = server.pathRE.exec("/2.0/12345/storage/crypto/keys");
- let [all, version, user, first, rest] = parts;
- do_check_eq(all, "/2.0/12345/storage/crypto/keys");
- do_check_eq(version, "2.0");
- do_check_eq(user, "12345");
- do_check_eq(first, "storage");
- do_check_eq(rest, "crypto/keys");
- do_check_eq(null, server.pathRE.exec("/nothing/else"));
-
- // Check that we can parse a collection URI.
- parts = server.pathRE.exec("/2.0/123/storage/crypto");
- [all, version, user, first, rest] = parts;
- do_check_eq(all, "/2.0/123/storage/crypto");
- do_check_eq(version, "2.0");
- do_check_eq(user, "123");
- do_check_eq(first, "storage");
- do_check_eq(rest, "crypto");
-
- // We don't allow trailing slash on storage URI.
- parts = server.pathRE.exec("/2.0/1234/storage/");
- do_check_eq(parts, undefined);
-
- // storage alone is a valid request.
- parts = server.pathRE.exec("/2.0/123456/storage");
- [all, version, user, first, rest] = parts;
- do_check_eq(all, "/2.0/123456/storage");
- do_check_eq(version, "2.0");
- do_check_eq(user, "123456");
- do_check_eq(first, "storage");
- do_check_eq(rest, undefined);
-
- parts = server.storageRE.exec("storage");
- let collection;
- [all, , collection, ] = parts;
- do_check_eq(all, "storage");
- do_check_eq(collection, undefined);
-
- run_next_test();
-});
-
-add_test(function test_basic_http() {
- let server = new StorageServer();
- server.registerUser("345", "password");
- do_check_true(server.userExists("345"));
- server.startSynchronous();
-
- _("Started on " + server.port);
- do_check_eq(server.requestCount, 0);
- let req = localRequest(server, "/2.0/storage/crypto/keys");
- _("req is " + req);
- req.get(function(err) {
- do_check_eq(null, err);
- do_check_eq(server.requestCount, 1);
- server.stop(run_next_test);
- });
-});
-
-add_test(function test_info_collections() {
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let path = "/2.0/123/info/collections";
-
- _("info/collections on empty server should be empty object.");
- let request = localRequest(server, path, "123", "password");
- let error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 200);
- do_check_eq(request.response.body, "{}");
-
- _("Creating an empty collection should result in collection appearing.");
- let coll = server.createCollection("123", "col1");
- request = localRequest(server, path, "123", "password");
- error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 200);
- let info = JSON.parse(request.response.body);
- do_check_attribute_count(info, 1);
- do_check_true("col1" in info);
- do_check_eq(info.col1, coll.timestamp);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_get_existing() {
- _("Ensure that BSO retrieval works.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- test: {"bso": {"foo": "bar"}}
- });
- server.startSynchronous();
-
- let coll = server.user("123").collection("test");
-
- let request = localRequest(server, "/2.0/123/storage/test/bso", "123",
- "password");
- let error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 200);
- do_check_eq(request.response.headers["content-type"], "application/json");
- let bso = JSON.parse(request.response.body);
- do_check_attribute_count(bso, 3);
- do_check_eq(bso.id, "bso");
- do_check_eq(bso.modified, coll.bso("bso").modified);
- let payload = JSON.parse(bso.payload);
- do_check_attribute_count(payload, 1);
- do_check_eq(payload.foo, "bar");
-
- server.stop(run_next_test);
-});
-
-add_test(function test_percent_decoding() {
- _("Ensure query string arguments with percent encoded are handled.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let coll = server.user("123").createCollection("test");
- coll.insert("001", {foo: "bar"});
- coll.insert("002", {bar: "foo"});
-
- let request = localRequest(server, "/2.0/123/storage/test?ids=001%2C002",
- "123", "password");
- let error = doGetRequest(request);
- do_check_null(error);
- do_check_eq(request.response.status, 200);
- let items = JSON.parse(request.response.body).items;
- do_check_attribute_count(items, 2);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_404() {
- _("Ensure the server responds with a 404 if a BSO does not exist.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- test: {}
- });
- server.startSynchronous();
-
- let request = localRequest(server, "/2.0/123/storage/test/foo");
- let error = doGetRequest(request);
- do_check_eq(error, null);
-
- do_check_eq(request.response.status, 404);
- do_check_false("content-type" in request.response.headers);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_if_modified_since_304() {
- _("Ensure the server responds properly to X-If-Modified-Since for BSOs.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- test: {bso: {foo: "bar"}}
- });
- server.startSynchronous();
-
- let coll = server.user("123").collection("test");
- do_check_neq(coll, null);
-
- // Rewind clock just in case.
- coll.timestamp -= 10000;
- coll.bso("bso").modified -= 10000;
-
- let request = localRequest(server, "/2.0/123/storage/test/bso",
- "123", "password");
- request.setHeader("X-If-Modified-Since", "" + server.serverTime());
- let error = doGetRequest(request);
- do_check_eq(null, error);
-
- do_check_eq(request.response.status, 304);
- do_check_false("content-type" in request.response.headers);
-
- request = localRequest(server, "/2.0/123/storage/test/bso",
- "123", "password");
- request.setHeader("X-If-Modified-Since", "" + (server.serverTime() - 20000));
- error = doGetRequest(request);
- do_check_eq(null, error);
- do_check_eq(request.response.status, 200);
- do_check_eq(request.response.headers["content-type"], "application/json");
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_if_unmodified_since() {
- _("Ensure X-If-Unmodified-Since works properly on BSOs.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- test: {bso: {foo: "bar"}}
- });
- server.startSynchronous();
-
- let coll = server.user("123").collection("test");
- do_check_neq(coll, null);
-
- let time = coll.bso("bso").modified;
-
- _("Ensure we get a 412 for specified times older than server time.");
- let request = localRequest(server, "/2.0/123/storage/test/bso",
- "123", "password");
- request.setHeader("X-If-Unmodified-Since", time - 5000);
- request.setHeader("Content-Type", "application/json");
- let payload = JSON.stringify({"payload": "foobar"});
- let error = doPutRequest(request, payload);
- do_check_eq(null, error);
- do_check_eq(request.response.status, 412);
-
- _("Ensure we get a 204 if update goes through.");
- request = localRequest(server, "/2.0/123/storage/test/bso",
- "123", "password");
- request.setHeader("Content-Type", "application/json");
- request.setHeader("X-If-Unmodified-Since", time + 1);
- error = doPutRequest(request, payload);
- do_check_eq(null, error);
- do_check_eq(request.response.status, 204);
- do_check_true(coll.timestamp > time);
-
- // Not sure why a client would send X-If-Unmodified-Since if a BSO doesn't
- // exist. But, why not test it?
- _("Ensure we get a 201 if creation goes through.");
- request = localRequest(server, "/2.0/123/storage/test/none",
- "123", "password");
- request.setHeader("Content-Type", "application/json");
- request.setHeader("X-If-Unmodified-Since", time);
- error = doPutRequest(request, payload);
- do_check_eq(null, error);
- do_check_eq(request.response.status, 201);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_delete_not_exist() {
- _("Ensure server behaves properly when deleting a BSO that does not exist.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.user("123").createCollection("empty");
- server.startSynchronous();
-
- server.callback.onItemDeleted = function onItemDeleted(username, collection,
- id) {
- do_throw("onItemDeleted should not have been called.");
- };
-
- let request = localRequest(server, "/2.0/123/storage/empty/nada",
- "123", "password");
- let error = doDeleteRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 404);
- do_check_false("content-type" in request.response.headers);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_delete_exists() {
- _("Ensure proper semantics when deleting a BSO that exists.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let coll = server.user("123").createCollection("test");
- coll.insert("myid", {foo: "bar"});
- let timestamp = coll.timestamp;
-
- server.callback.onItemDeleted = function onDeleted(username, collection, id) {
- delete server.callback.onItemDeleted;
- do_check_eq(username, "123");
- do_check_eq(collection, "test");
- do_check_eq(id, "myid");
- };
-
- let request = localRequest(server, "/2.0/123/storage/test/myid",
- "123", "password");
- let error = doDeleteRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 204);
- do_check_eq(coll.bsos().length, 0);
- do_check_true(coll.timestamp > timestamp);
-
- _("On next request the BSO should not exist.");
- request = localRequest(server, "/2.0/123/storage/test/myid",
- "123", "password");
- error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 404);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_delete_unmodified() {
- _("Ensure X-If-Unmodified-Since works when deleting BSOs.");
-
- let server = new StorageServer();
- server.startSynchronous();
- server.registerUser("123", "password");
- let coll = server.user("123").createCollection("test");
- let bso = coll.insert("myid", {foo: "bar"});
-
- let modified = bso.modified;
-
- _("Issuing a DELETE with an older time should fail.");
- let path = "/2.0/123/storage/test/myid";
- let request = localRequest(server, path, "123", "password");
- request.setHeader("X-If-Unmodified-Since", modified - 1000);
- let error = doDeleteRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 412);
- do_check_false("content-type" in request.response.headers);
- do_check_neq(coll.bso("myid"), null);
-
- _("Issuing a DELETE with a newer time should work.");
- request = localRequest(server, path, "123", "password");
- request.setHeader("X-If-Unmodified-Since", modified + 1000);
- error = doDeleteRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 204);
- do_check_true(coll.bso("myid").deleted);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_collection_get_unmodified_since() {
- _("Ensure conditional unmodified get on collection works when it should.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
- let collection = server.user("123").createCollection("testcoll");
- collection.insert("bso0", {foo: "bar"});
-
- let serverModified = collection.timestamp;
-
- let request1 = localRequest(server, "/2.0/123/storage/testcoll",
- "123", "password");
- request1.setHeader("X-If-Unmodified-Since", serverModified);
- let error = doGetRequest(request1);
- do_check_null(error);
- do_check_eq(request1.response.status, 200);
-
- let request2 = localRequest(server, "/2.0/123/storage/testcoll",
- "123", "password");
- request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
- error = doGetRequest(request2);
- do_check_null(error);
- do_check_eq(request2.response.status, 412);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_bso_get_unmodified_since() {
- _("Ensure conditional unmodified get on BSO works appropriately.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
- let collection = server.user("123").createCollection("testcoll");
- let bso = collection.insert("bso0", {foo: "bar"});
-
- let serverModified = bso.modified;
-
- let request1 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
- "123", "password");
- request1.setHeader("X-If-Unmodified-Since", serverModified);
- let error = doGetRequest(request1);
- do_check_null(error);
- do_check_eq(request1.response.status, 200);
-
- let request2 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
- "123", "password");
- request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
- error = doGetRequest(request2);
- do_check_null(error);
- do_check_eq(request2.response.status, 412);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_missing_collection_404() {
- _("Ensure a missing collection returns a 404.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let request = localRequest(server, "/2.0/123/storage/none", "123", "password");
- let error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 404);
- do_check_false("content-type" in request.response.headers);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_get_storage_405() {
- _("Ensure that a GET on /storage results in a 405.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let request = localRequest(server, "/2.0/123/storage", "123", "password");
- let error = doGetRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 405);
- do_check_eq(request.response.headers.allow, "DELETE");
-
- server.stop(run_next_test);
-});
-
-add_test(function test_delete_storage() {
- _("Ensure that deleting all of storage works.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- foo: {a: {foo: "bar"}, b: {bar: "foo"}},
- baz: {c: {bob: "law"}, blah: {law: "blog"}}
- });
-
- server.startSynchronous();
-
- let request = localRequest(server, "/2.0/123/storage", "123", "password");
- let error = doDeleteRequest(request);
- do_check_eq(error, null);
- do_check_eq(request.response.status, 204);
- do_check_attribute_count(server.users["123"].collections, 0);
-
- server.stop(run_next_test);
-});
-
-add_test(function test_x_num_records() {
- let server = new StorageServer();
- server.registerUser("123", "password");
-
- server.createContents("123", {
- crypto: {foos: {foo: "bar"},
- bars: {foo: "baz"}}
- });
- server.startSynchronous();
- let bso = localRequest(server, "/2.0/123/storage/crypto/foos");
- bso.get(function(err) {
- // BSO fetches don't have one.
- do_check_false("x-num-records" in this.response.headers);
- let col = localRequest(server, "/2.0/123/storage/crypto");
- col.get(function(err2) {
- // Collection fetches do.
- do_check_eq(this.response.headers["x-num-records"], "2");
- server.stop(run_next_test);
- });
- });
-});
-
-add_test(function test_put_delete_put() {
- _("Bug 790397: Ensure BSO deleted flag is reset on PUT.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.createContents("123", {
- test: {bso: {foo: "bar"}}
- });
- server.startSynchronous();
-
- _("Ensure we can PUT an existing record.");
- let request1 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
- request1.setHeader("Content-Type", "application/json");
- let payload1 = JSON.stringify({"payload": "foobar"});
- let error1 = doPutRequest(request1, payload1);
- do_check_eq(null, error1);
- do_check_eq(request1.response.status, 204);
-
- _("Ensure we can DELETE it.");
- let request2 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
- let error2 = doDeleteRequest(request2);
- do_check_eq(error2, null);
- do_check_eq(request2.response.status, 204);
- do_check_false("content-type" in request2.response.headers);
-
- _("Ensure we can PUT a previously deleted record.");
- let request3 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
- request3.setHeader("Content-Type", "application/json");
- let payload3 = JSON.stringify({"payload": "foobar"});
- let error3 = doPutRequest(request3, payload3);
- do_check_eq(null, error3);
- do_check_eq(request3.response.status, 201);
-
- _("Ensure we can GET the re-uploaded record.");
- let request4 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
- let error4 = doGetRequest(request4);
- do_check_eq(error4, null);
- do_check_eq(request4.response.status, 200);
- do_check_eq(request4.response.headers["content-type"], "application/json");
-
- server.stop(run_next_test);
-});
-
-add_test(function test_collection_get_newer() {
- _("Ensure get with newer argument on collection works.");
-
- let server = new StorageServer();
- server.registerUser("123", "password");
- server.startSynchronous();
-
- let coll = server.user("123").createCollection("test");
- let bso1 = coll.insert("001", {foo: "bar"});
- let bso2 = coll.insert("002", {bar: "foo"});
-
- // Don't want both records to have the same timestamp.
- bso2.modified = bso1.modified + 1000;
-
- function newerRequest(newer) {
- return localRequest(server, "/2.0/123/storage/test?newer=" + newer,
- "123", "password");
- }
-
- let request1 = newerRequest(0);
- let error1 = doGetRequest(request1);
- do_check_null(error1);
- do_check_eq(request1.response.status, 200);
- let items1 = JSON.parse(request1.response.body).items;
- do_check_attribute_count(items1, 2);
-
- let request2 = newerRequest(bso1.modified + 1);
- let error2 = doGetRequest(request2);
- do_check_null(error2);
- do_check_eq(request2.response.status, 200);
- let items2 = JSON.parse(request2.response.body).items;
- do_check_attribute_count(items2, 1);
-
- let request3 = newerRequest(bso2.modified + 1);
- let error3 = doGetRequest(request3);
- do_check_null(error3);
- do_check_eq(request3.response.status, 200);
- let items3 = JSON.parse(request3.response.body).items;
- do_check_attribute_count(items3, 0);
-
- server.stop(run_next_test);
-});
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@@ -53,12 +53,9 @@ skip-if = os == "android"
[test_restrequest.js]
[test_tokenauthenticatedrequest.js]
skip-if = os == "android"
[test_tokenserverclient.js]
skip-if = os == "android"
-[test_storage_server.js]
-skip-if = os == "android"
-
[test_uptake_telemetry.js]
