--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -5118,17 +5118,17 @@ pref("browser.safebrowsing.provider.goog
pref("browser.safebrowsing.provider.google.lists", "goog-badbinurl-shavar,goog-downloadwhite-digest256,goog-phish-shavar,googpub-phish-shavar,goog-malware-shavar,goog-unwanted-shavar");
pref("browser.safebrowsing.provider.google.updateURL", "https://safebrowsing.google.com/safebrowsing/downloads?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2&key=%GOOGLE_API_KEY%");
pref("browser.safebrowsing.provider.google.gethashURL", "https://safebrowsing.google.com/safebrowsing/gethash?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2");
pref("browser.safebrowsing.provider.google.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
// Prefs for v4.
pref("browser.safebrowsing.provider.google4.pver", "4");
pref("browser.safebrowsing.provider.google4.lists", "goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto");
-pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$req=%REQUEST_BASE64%&$ct=application/x-protobuf&key=%GOOGLE_API_KEY%");
+pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%");
pref("browser.safebrowsing.provider.google4.gethashURL", "https://safebrowsing.googleapis.com/v4/fullHashes:find?$req=%REQUEST_BASE64%&$ct=application/x-protobuf&key=%GOOGLE_API_KEY%");
pref("browser.safebrowsing.provider.google4.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
pref("browser.safebrowsing.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.reportPhishURL", "https://%LOCALE%.phish-report.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
// The table and global pref for blocking plugin content
--- a/netwerk/test/unit/test_cookiejars_safebrowsing.js
+++ b/netwerk/test/unit/test_cookiejars_safebrowsing.js
@@ -110,17 +110,17 @@ add_test(function test_safebrowsing_upda
function onUpdateError() {
do_throw("ERROR: received onUpdateError!");
}
function onDownloadError() {
do_throw("ERROR: received onDownloadError!");
}
streamUpdater.downloadUpdates("test-phish-simple,test-malware-simple", "",
- URL + safebrowsingUpdatePath, onSuccess, onUpdateError, onDownloadError);
+ true, URL + safebrowsingUpdatePath, onSuccess, onUpdateError, onDownloadError);
});
add_test(function test_non_safebrowsing_cookie() {
var cookieName = 'regCookie_id0';
var loadContext = new LoadContextCallback(0, false, false, false);
function setNonSafeBrowsingCookie() {
--- a/toolkit/components/downloads/test/unit/test_app_rep.js
+++ b/toolkit/components/downloads/test/unit/test_app_rep.js
@@ -215,16 +215,17 @@ add_test(function test_local_list() {
}
// Just throw if we ever get an update or download error.
function handleError(aEvent) {
do_throw("We didn't download or update correctly: " + aEvent);
}
streamUpdater.downloadUpdates(
"goog-downloadwhite-digest256,goog-badbinurl-shavar",
"goog-downloadwhite-digest256,goog-badbinurl-shavar;\n",
+ true, // isPostRequest.
"http://localhost:4444/downloads",
updateSuccess, handleError, handleError);
});
add_test(function test_unlisted() {
Services.prefs.setCharPref(appRepURLPref,
"http://localhost:4444/download");
let counts = get_telemetry_counts();
--- a/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js
+++ b/toolkit/components/downloads/test/unit/test_app_rep_maclinux.js
@@ -192,16 +192,17 @@ function waitForUpdates() {
// Just throw if we ever get an update or download error.
function handleError(aEvent) {
do_throw("We didn't download or update correctly: " + aEvent);
deferred.reject();
}
streamUpdater.downloadUpdates(
"goog-downloadwhite-digest256",
"goog-downloadwhite-digest256;\n",
+ true,
"http://localhost:4444/downloads",
updateSuccess, handleError, handleError);
return deferred.promise;
}
function promiseQueryReputation(query, expectedShouldBlock) {
let deferred = Promise.defer();
function onComplete(aShouldBlock, aStatus) {
--- a/toolkit/components/downloads/test/unit/test_app_rep_windows.js
+++ b/toolkit/components/downloads/test/unit/test_app_rep_windows.js
@@ -292,16 +292,17 @@ function waitForUpdates() {
// Just throw if we ever get an update or download error.
function handleError(aEvent) {
do_throw("We didn't download or update correctly: " + aEvent);
deferred.reject();
}
streamUpdater.downloadUpdates(
"goog-downloadwhite-digest256",
"goog-downloadwhite-digest256;\n",
+ true,
"http://localhost:4444/downloads",
updateSuccess, handleError, handleError);
return deferred.promise;
}
function promiseQueryReputation(query, expectedShouldBlock) {
let deferred = Promise.defer();
function onComplete(aShouldBlock, aStatus) {
--- a/toolkit/components/url-classifier/content/listmanager.js
+++ b/toolkit/components/url-classifier/content/listmanager.js
@@ -7,17 +7,17 @@ Cu.import("resource://gre/modules/Servic
// This is the only implementation of nsIUrlListManager.
// A class that manages lists, namely white and black lists for
// phishing or malware protection. The ListManager knows how to fetch,
// update, and store lists.
//
// There is a single listmanager for the whole application.
//
-// TODO more comprehensive update tests, for example add unittest check
+// TODO more comprehensive update tests, for example add unittest check
// that the listmanagers tables are properly written on updates
// Lower and upper limits on the server-provided polling frequency
const minDelayMs = 5 * 60 * 1000;
const maxDelayMs = 24 * 60 * 60 * 1000;
// Log only if browser.safebrowsing.debug is true
this.log = function log(...stuff) {
@@ -347,91 +347,119 @@ PROT_ListManager.prototype.makeUpdateReq
if (!updateUrl) {
return;
}
// An object of the form
// { tableList: comma-separated list of tables to request,
// tableNames: map of tables that need updating,
// request: list of tables and existing chunk ranges from tableData
// }
- var streamerMap = { tableList: null, tableNames: {}, request: "" };
+ var streamerMap = { tableList: null,
+ tableNames: {},
+ requestPayload: "",
+ isPostRequest: true };
+
let useProtobuf = false;
+ let onceThru = false;
for (var tableName in this.tablesData) {
// Skip tables not matching this update url
if (this.tablesData[tableName].updateUrl != updateUrl) {
continue;
}
// Check if |updateURL| is for 'proto'. (only v4 uses protobuf for now.)
// We use the table name 'goog-*-proto' and an additional provider "google4"
// to describe the v4 settings.
let isCurTableProto = tableName.endsWith('-proto');
- if (useProtobuf && !isCurTableProto) {
- log('ERROR: Tables for the same updateURL should all be "proto" or none. ' +
- 'Check "browser.safebrowsing.provider.google4.lists"');
+ if (!onceThru) {
+ useProtobuf = isCurTableProto;
+ onceThru = true;
+ } else if (useProtobuf !== isCurTableProto) {
+ log('ERROR: Cannot mix "proto" tables with other types ' +
+ 'within the same provider.');
}
- useProtobuf = isCurTableProto;
if (this.needsUpdate_[this.tablesData[tableName].updateUrl][tableName]) {
streamerMap.tableNames[tableName] = true;
}
if (!streamerMap.tableList) {
streamerMap.tableList = tableName;
} else {
streamerMap.tableList += "," + tableName;
}
}
if (useProtobuf) {
- // TODO: Bug 1275507 - XPCOM API to build v4 update request.
- streamerMap.request = "";
+ let tableArray = streamerMap.tableList.split(',');
+
+ // The state is a byte stream which server told us from the
+ // last table update. The state would be used to do the partial
+ // update and the empty string means the table has
+ // never been downloaded. See Bug 1287058 for supporting
+ // partial update.
+ let stateArray = [];
+ tableArray.forEach(() => stateArray.push(''));
+
+ let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+ .getService(Ci.nsIUrlClassifierUtils);
+ let requestPayload = urlUtils.makeUpdateRequestV4(tableArray,
+ stateArray,
+ tableArray.length);
+ // Use a base64-encoded request.
+ streamerMap.requestPayload = btoa(requestPayload);
+ streamerMap.isPostRequest = false;
} else {
// Build the request. For each table already in the database, include the
// chunk data from the database
var lines = tableData.split("\n");
for (var i = 0; i < lines.length; i++) {
var fields = lines[i].split(";");
var name = fields[0];
if (streamerMap.tableNames[name]) {
- streamerMap.request += lines[i] + "\n";
+ streamerMap.requestPayload += lines[i] + "\n";
delete streamerMap.tableNames[name];
}
}
// For each requested table that didn't have chunk data in the database,
// request it fresh
for (let tableName in streamerMap.tableNames) {
- streamerMap.request += tableName + ";\n";
+ streamerMap.requestPayload += tableName + ";\n";
}
+
+ streamerMap.isPostRequest = true;
}
log("update request: " + JSON.stringify(streamerMap, undefined, 2) + "\n");
// Don't send an empty request.
- if (streamerMap.request.length > 0) {
+ if (streamerMap.requestPayload.length > 0) {
this.makeUpdateRequestForEntry_(updateUrl, streamerMap.tableList,
- streamerMap.request);
+ streamerMap.requestPayload,
+ streamerMap.isPostRequest);
} else {
// We were disabled between kicking off getTables and now.
log("Not sending empty request");
}
}
PROT_ListManager.prototype.makeUpdateRequestForEntry_ = function(updateUrl,
tableList,
- request) {
- log("makeUpdateRequestForEntry_: request " + request +
+ requestPayload,
+ isPostRequest) {
+ log("makeUpdateRequestForEntry_: requestPayload " + requestPayload +
" update: " + updateUrl + " tablelist: " + tableList + "\n");
var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
.getService(Ci.nsIUrlClassifierStreamUpdater);
this.requestBackoffs_[updateUrl].noteRequest();
if (!streamer.downloadUpdates(
tableList,
- request,
+ requestPayload,
+ isPostRequest,
updateUrl,
BindToObject(this.updateSuccess_, this, tableList, updateUrl),
BindToObject(this.updateError_, this, tableList, updateUrl),
BindToObject(this.downloadError_, this, tableList, updateUrl))) {
// Our alarm gets reset in one of the 3 callbacks.
log("pending update, queued request until later");
}
}
--- a/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl
+++ b/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl
@@ -15,22 +15,25 @@
interface nsIUrlClassifierStreamUpdater : nsISupports
{
/**
* Try to download updates from updateUrl. If an update is already in
* progress, queues the requested update. This is used in nsIUrlListManager
* as well as in testing.
* @param aRequestTables Comma-separated list of tables included in this
* update.
- * @param aRequestBody The body for the request.
+ * @param aRequestPayload The payload for the request.
+ * @param aIsPostRequest Whether the request should be sent by POST method.
+ * Should be 'true' for v2 usage.
* @param aUpdateUrl The plaintext url from which to request updates.
* @param aSuccessCallback Called after a successful update.
* @param aUpdateErrorCallback Called for problems applying the update
* @param aDownloadErrorCallback Called if we get an http error or a
* connection refused error.
*/
boolean downloadUpdates(in ACString aRequestTables,
- in ACString aRequestBody,
+ in ACString aRequestPayload,
+ in boolean aIsPostRequest,
in ACString aUpdateUrl,
in nsIUrlClassifierCallback aSuccessCallback,
in nsIUrlClassifierCallback aUpdateErrorCallback,
in nsIUrlClassifierCallback aDownloadErrorCallback);
};
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp
@@ -98,25 +98,26 @@ nsUrlClassifierStreamUpdater::DownloadDo
mDownloadErrorCallback = nullptr;
}
///////////////////////////////////////////////////////////////////////////////
// nsIUrlClassifierStreamUpdater implementation
nsresult
nsUrlClassifierStreamUpdater::FetchUpdate(nsIURI *aUpdateUrl,
- const nsACString & aRequestBody,
+ const nsACString & aRequestPayload,
+ bool aIsPostRequest,
const nsACString & aStreamTable)
{
#ifdef DEBUG
{
nsCString spec;
aUpdateUrl->GetSpec(spec);
- LOG(("Fetching update %s from %s", aRequestBody.Data(), spec.get()));
+ LOG(("Fetching update %s from %s", aRequestPayload.Data(), spec.get()));
}
#endif
nsresult rv;
uint32_t loadFlags = nsIChannel::INHIBIT_CACHING |
nsIChannel::LOAD_BYPASS_CACHE;
rv = NS_NewChannel(getter_AddRefs(mChannel),
aUpdateUrl,
@@ -129,19 +130,36 @@ nsUrlClassifierStreamUpdater::FetchUpdat
NS_ENSURE_SUCCESS(rv, rv);
nsCOMPtr<nsILoadInfo> loadInfo = mChannel->GetLoadInfo();
loadInfo->SetOriginAttributes(mozilla::NeckoOriginAttributes(NECKO_SAFEBROWSING_APP_ID, false));
mBeganStream = false;
- // If aRequestBody is empty, construct it for the test.
- if (!aRequestBody.IsEmpty()) {
- rv = AddRequestBody(aRequestBody);
+ if (!aIsPostRequest) {
+ // We use POST method to send our request in v2. In v4, the request
+ // needs to be embedded to the URL and use GET method to send.
+ // However, from the Chromium source code, a extended HTTP header has
+ // to be sent along with the request to make the request succeed.
+ // The following description is from Chromium source code:
+ //
+ // "The following header informs the envelope server (which sits in
+ // front of Google's stubby server) that the received GET request should be
+ // interpreted as a POST."
+ //
+ nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("X-HTTP-Method-Override"),
+ NS_LITERAL_CSTRING("POST"),
+ false);
+ NS_ENSURE_SUCCESS(rv, rv);
+ } else if (!aRequestPayload.IsEmpty()) {
+ rv = AddRequestBody(aRequestPayload);
NS_ENSURE_SUCCESS(rv, rv);
}
// Set the appropriate content type for file/data URIs, for unit testing
// purposes.
// This is only used for testing and should be deleted.
bool match;
if ((NS_SUCCEEDED(aUpdateUrl->SchemeIs("file", &match)) && match) ||
@@ -171,54 +189,62 @@ nsUrlClassifierStreamUpdater::FetchUpdat
mStreamTable = aStreamTable;
return NS_OK;
}
nsresult
nsUrlClassifierStreamUpdater::FetchUpdate(const nsACString & aUpdateUrl,
- const nsACString & aRequestBody,
+ const nsACString & aRequestPayload,
+ bool aIsPostRequest,
const nsACString & aStreamTable)
{
LOG(("(pre) Fetching update from %s\n", PromiseFlatCString(aUpdateUrl).get()));
+ nsCString updateUrl(aUpdateUrl);
+ if (!aIsPostRequest) {
+ updateUrl.AppendPrintf("&$req=%s", nsCString(aRequestPayload).get());
+ }
+
nsCOMPtr<nsIURI> uri;
- nsresult rv = NS_NewURI(getter_AddRefs(uri), aUpdateUrl);
+ nsresult rv = NS_NewURI(getter_AddRefs(uri), updateUrl);
NS_ENSURE_SUCCESS(rv, rv);
nsAutoCString urlSpec;
uri->GetAsciiSpec(urlSpec);
LOG(("(post) Fetching update from %s\n", urlSpec.get()));
- return FetchUpdate(uri, aRequestBody, aStreamTable);
+ return FetchUpdate(uri, aRequestPayload, aIsPostRequest, aStreamTable);
}
NS_IMETHODIMP
nsUrlClassifierStreamUpdater::DownloadUpdates(
const nsACString &aRequestTables,
- const nsACString &aRequestBody,
+ const nsACString &aRequestPayload,
+ bool aIsPostRequest,
const nsACString &aUpdateUrl,
nsIUrlClassifierCallback *aSuccessCallback,
nsIUrlClassifierCallback *aUpdateErrorCallback,
nsIUrlClassifierCallback *aDownloadErrorCallback,
bool *_retval)
{
NS_ENSURE_ARG(aSuccessCallback);
NS_ENSURE_ARG(aUpdateErrorCallback);
NS_ENSURE_ARG(aDownloadErrorCallback);
if (mIsUpdating) {
- LOG(("Already updating, queueing update %s from %s", aRequestBody.Data(),
+ LOG(("Already updating, queueing update %s from %s", aRequestPayload.Data(),
aUpdateUrl.Data()));
*_retval = false;
PendingRequest *request = mPendingRequests.AppendElement();
request->mTables = aRequestTables;
- request->mRequest = aRequestBody;
+ request->mRequestPayload = aRequestPayload;
+ request->mIsPostRequest = aIsPostRequest;
request->mUrl = aUpdateUrl;
request->mSuccessCallback = aSuccessCallback;
request->mUpdateErrorCallback = aUpdateErrorCallback;
request->mDownloadErrorCallback = aDownloadErrorCallback;
return NS_OK;
}
if (aUpdateUrl.IsEmpty()) {
@@ -243,21 +269,22 @@ nsUrlClassifierStreamUpdater::DownloadUp
NS_ENSURE_SUCCESS(rv, rv);
mInitialized = true;
}
rv = mDBService->BeginUpdate(this, aRequestTables);
if (rv == NS_ERROR_NOT_AVAILABLE) {
LOG(("Service busy, already updating, queuing update %s from %s",
- aRequestBody.Data(), aUpdateUrl.Data()));
+ aRequestPayload.Data(), aUpdateUrl.Data()));
*_retval = false;
PendingRequest *request = mPendingRequests.AppendElement();
request->mTables = aRequestTables;
- request->mRequest = aRequestBody;
+ request->mRequestPayload = aRequestPayload;
+ request->mIsPostRequest = aIsPostRequest;
request->mUrl = aUpdateUrl;
request->mSuccessCallback = aSuccessCallback;
request->mUpdateErrorCallback = aUpdateErrorCallback;
request->mDownloadErrorCallback = aDownloadErrorCallback;
return NS_OK;
}
if (NS_FAILED(rv)) {
@@ -267,19 +294,18 @@ nsUrlClassifierStreamUpdater::DownloadUp
mSuccessCallback = aSuccessCallback;
mUpdateErrorCallback = aUpdateErrorCallback;
mDownloadErrorCallback = aDownloadErrorCallback;
mIsUpdating = true;
*_retval = true;
LOG(("FetchUpdate: %s", aUpdateUrl.Data()));
- //LOG(("requestBody: %s", aRequestBody.Data()));
- return FetchUpdate(aUpdateUrl, aRequestBody, EmptyCString());
+ return FetchUpdate(aUpdateUrl, aRequestPayload, aIsPostRequest, EmptyCString());
}
///////////////////////////////////////////////////////////////////////////////
// nsIUrlClassifierUpdateObserver implementation
NS_IMETHODIMP
nsUrlClassifierStreamUpdater::UpdateUrlRequested(const nsACString &aUrl,
const nsACString &aTable)
@@ -313,17 +339,19 @@ nsresult
nsUrlClassifierStreamUpdater::FetchNext()
{
if (mPendingUpdates.Length() == 0) {
return NS_OK;
}
PendingUpdate &update = mPendingUpdates[0];
LOG(("Fetching update url: %s\n", update.mUrl.get()));
- nsresult rv = FetchUpdate(update.mUrl, EmptyCString(),
+ nsresult rv = FetchUpdate(update.mUrl,
+ EmptyCString(),
+ true, // This method is for v2 and v2 is always a POST.
update.mTable);
if (NS_FAILED(rv)) {
LOG(("Error fetching update url: %s\n", update.mUrl.get()));
// We can commit the urls that we've applied so far. This is
// probably a transient server problem, so trigger backoff.
mDownloadErrorCallback->HandleEvent(EmptyCString());
mDownloadError = true;
mDBService->FinishUpdate();
@@ -344,17 +372,18 @@ nsUrlClassifierStreamUpdater::FetchNextR
}
PendingRequest &request = mPendingRequests[0];
LOG(("Stream updater: fetching next request: %s, %s",
request.mTables.get(), request.mUrl.get()));
bool dummy;
DownloadUpdates(
request.mTables,
- request.mRequest,
+ request.mRequestPayload,
+ request.mIsPostRequest,
request.mUrl,
request.mSuccessCallback,
request.mUpdateErrorCallback,
request.mDownloadErrorCallback,
&dummy);
request.mSuccessCallback = nullptr;
request.mUpdateErrorCallback = nullptr;
request.mDownloadErrorCallback = nullptr;
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h
+++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h
@@ -49,21 +49,23 @@ private:
// Disallow copy constructor
nsUrlClassifierStreamUpdater(nsUrlClassifierStreamUpdater&);
nsresult AddRequestBody(const nsACString &aRequestBody);
// Fetches an update for a single table.
nsresult FetchUpdate(nsIURI *aURI,
- const nsACString &aRequestBody,
+ const nsACString &aRequest,
+ bool aIsPostRequest,
const nsACString &aTable);
// Dumb wrapper so we don't have to create URIs.
nsresult FetchUpdate(const nsACString &aURI,
- const nsACString &aRequestBody,
+ const nsACString &aRequest,
+ bool aIsPostRequest,
const nsACString &aTable);
// Fetches the next table, from mPendingUpdates.
nsresult FetchNext();
// Fetches the next request, from mPendingRequests
nsresult FetchNextRequest();
@@ -73,17 +75,18 @@ private:
bool mBeganStream;
nsCString mStreamTable;
nsCOMPtr<nsIChannel> mChannel;
nsCOMPtr<nsIUrlClassifierDBService> mDBService;
nsCOMPtr<nsITimer> mTimer;
struct PendingRequest {
nsCString mTables;
- nsCString mRequest;
+ nsCString mRequestPayload;
+ bool mIsPostRequest;
nsCString mUrl;
nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback;
nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback;
nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback;
};
nsTArray<PendingRequest> mPendingRequests;
struct PendingUpdate {
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
@@ -206,16 +206,19 @@ nsUrlClassifierUtils::GetKeyForURI(nsIUR
static const struct {
const char* mListName;
uint32_t mThreatType;
} THREAT_TYPE_CONV_TABLE[] = {
{ "goog-malware-proto", MALWARE_THREAT}, // 1
{ "googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2
{ "goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3
{ "goog-phish-proto", SOCIAL_ENGINEERING}, // 5
+
+ // For testing purpose.
+ { "test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2
};
NS_IMETHODIMP
nsUrlClassifierUtils::ConvertThreatTypeToListName(uint32_t aThreatType,
nsACString& aListName)
{
for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) {
if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) {
--- a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
+++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
@@ -195,17 +195,17 @@ function doErrorUpdate(tables, success,
*/
function doStreamUpdate(updateText, success, failure, downloadFailure) {
var dataUpdate = "data:," + encodeURIComponent(updateText);
if (!downloadFailure) {
downloadFailure = failure;
}
- streamUpdater.downloadUpdates(allTables, "",
+ streamUpdater.downloadUpdates(allTables, "", true,
dataUpdate, success, failure, downloadFailure);
}
var gAssertions = {
tableData : function(expectedTables, cb)
{
dbservice.getTables(function(tables) {
--- a/toolkit/components/url-classifier/tests/unit/test_digest256.js
+++ b/toolkit/components/url-classifier/tests/unit/test_digest256.js
@@ -113,16 +113,17 @@ add_test(function test_update() {
// passed back in the callback in nsIUrlClassifierStreamUpdater on success.
do_check_eq("1000", aEvent);
do_print("All data processed");
run_next_test();
}
streamUpdater.downloadUpdates(
"goog-downloadwhite-digest256",
"goog-downloadwhite-digest256;\n",
+ true,
"http://localhost:4444/downloads",
updateSuccess, handleError, handleError);
});
add_test(function test_url_not_whitelisted() {
let uri = createURI("http://example.com");
let principal = gSecMan.createCodebasePrincipal(uri, {});
gDbService.lookup(principal, "goog-downloadwhite-digest256",
--- a/toolkit/components/url-classifier/tests/unit/test_listmanager.js
+++ b/toolkit/components/url-classifier/tests/unit/test_listmanager.js
@@ -26,55 +26,61 @@ const TEST_TABLE_DATA_LIST = [
{
tableName: "test-listmanager2-digest256",
providerName: "google",
updateUrl: "http://localhost:4444/safebrowsing/update",
gethashUrl: "http://localhost:4444/safebrowsing/gethash2",
}
];
-// This table has a different update URL.
-const TEST_TABLE_DATA_ANOTHER = {
- tableName: "test-listmanageranother-digest256",
- providerName: "google",
- updateUrl: "http://localhost:5555/safebrowsing/update",
- gethashUrl: "http://localhost:5555/safebrowsing/gethash-another",
+// This table has a different update URL (for v4).
+const TEST_TABLE_DATA_V4 = {
+ tableName: "test-phish-proto",
+ providerName: "google4",
+ updateUrl: "http://localhost:5555/safebrowsing/update?",
+ gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4",
};
const PREF_NEXTUPDATETIME = "browser.safebrowsing.provider.google.nextupdatetime";
+const PREF_NEXTUPDATETIME_V4 = "browser.safebrowsing.provider.google4.nextupdatetime";
let gListManager = Cc["@mozilla.org/url-classifier/listmanager;1"]
.getService(Ci.nsIUrlListManager);
+let gUrlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+ .getService(Ci.nsIUrlClassifierUtils);
+
// Global test server for serving safebrowsing updates.
let gHttpServ = null;
let gUpdateResponse = "";
let gExpectedUpdateRequest = "";
+let gExpectedQueryV4 = "";
-// Handles request for TEST_TABLE_DATA_ANOTHER.
-let gHttpServAnother = null;
+// Handles request for TEST_TABLE_DATA_V4.
+let gHttpServV4 = null;
// These two variables are used to synchronize the last two racing updates
// (in terms of "update URL") in test_update_all_tables().
let gUpdatedCntForTableData = 0; // For TEST_TABLE_DATA_LIST.
-let gIsAnotherUpdated = false; // For TEST_TABLE_DATA_ANOTHER.
+let gIsV4Updated = false; // For TEST_TABLE_DATA_V4.
prefBranch.setBoolPref("browser.safebrowsing.debug", true);
// Register tables.
TEST_TABLE_DATA_LIST.forEach(function(t) {
gListManager.registerTable(t.tableName,
t.providerName,
t.updateUrl,
t.gethashUrl);
});
-gListManager.registerTable(TEST_TABLE_DATA_ANOTHER.tableName,
- TEST_TABLE_DATA_ANOTHER.providerName,
- TEST_TABLE_DATA_ANOTHER.updateUrl,
- TEST_TABLE_DATA_ANOTHER.gethashUrl);
+
+gListManager.registerTable(TEST_TABLE_DATA_V4.tableName,
+ TEST_TABLE_DATA_V4.providerName,
+ TEST_TABLE_DATA_V4.updateUrl,
+ TEST_TABLE_DATA_V4.gethashUrl);
const SERVER_INVOLVED_TEST_CASE_LIST = [
// - Do table0 update.
// - Server would respond "a:5:32:32\n[DATA]".
function test_update_table0() {
disableAllUpdates();
gListManager.enableUpdate(TEST_TABLE_DATA_LIST[0].tableName);
@@ -105,41 +111,50 @@ const SERVER_INVOLVED_TEST_CASE_LIST = [
// - Server would respond no chunk control.
//
// Note that this test MUST be the last one in the array since we rely on
// the number of sever-involved test case to synchronize the racing last
// two udpates for different URL.
function test_update_all_tables() {
disableAllUpdates();
- // Enable all tables including TEST_TABLE_DATA_ANOTHER!
+ // Enable all tables including TEST_TABLE_DATA_V4!
TEST_TABLE_DATA_LIST.forEach(function(t) {
gListManager.enableUpdate(t.tableName);
});
- gListManager.enableUpdate(TEST_TABLE_DATA_ANOTHER.tableName);
+ gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName);
+ // Expected results for v2.
gExpectedUpdateRequest = TEST_TABLE_DATA_LIST[0].tableName + ";a:5:s:2-12\n" +
TEST_TABLE_DATA_LIST[1].tableName + ";\n" +
TEST_TABLE_DATA_LIST[2].tableName + ";\n";
gUpdateResponse = "n:1000\n";
+ // We test the request against the query string since v4 request
+ // would be appened to the query string. The request is generated
+ // by protobuf API (binary) then encoded to base64 format.
+ let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
+ [""],
+ 1);
+ gExpectedQueryV4 = "&$req=" + btoa(requestV4);
+
forceTableUpdate();
},
];
SERVER_INVOLVED_TEST_CASE_LIST.forEach(t => add_test(t));
// Tests nsIUrlListManager.getGethashUrl.
add_test(function test_getGethashUrl() {
TEST_TABLE_DATA_LIST.forEach(function (t) {
equal(gListManager.getGethashUrl(t.tableName), t.gethashUrl);
});
- equal(gListManager.getGethashUrl(TEST_TABLE_DATA_ANOTHER.tableName),
- TEST_TABLE_DATA_ANOTHER.gethashUrl);
+ equal(gListManager.getGethashUrl(TEST_TABLE_DATA_V4.tableName),
+ TEST_TABLE_DATA_V4.gethashUrl);
run_next_test();
});
function run_test() {
// Setup primary testing server.
gHttpServ = new HttpServer();
gHttpServ.registerDirectory("/", do_get_cwd());
@@ -160,71 +175,79 @@ function run_test() {
if (gUpdatedCntForTableData !== SERVER_INVOLVED_TEST_CASE_LIST.length) {
// This is not the last test case so run the next once upon the
// the update success.
waitForUpdateSuccess(run_next_test);
return;
}
- if (gIsAnotherUpdated) {
+ if (gIsV4Updated) {
run_next_test(); // All tests are done. Just finish.
return;
}
- do_print("Waiting for TEST_TABLE_DATA_ANOTHER to be tested ...");
+ do_print("Waiting for TEST_TABLE_DATA_V4 to be tested ...");
});
gHttpServ.start(4444);
- // Setup another testing server for the different update URL.
- gHttpServAnother = new HttpServer();
- gHttpServAnother.registerDirectory("/", do_get_cwd());
+ // Setup v4 testing server for the different update URL.
+ gHttpServV4 = new HttpServer();
+ gHttpServV4.registerDirectory("/", do_get_cwd());
+
+ gHttpServV4.registerPathHandler("/safebrowsing/update", function(request, response) {
+ // V4 update request body should be empty.
+ equal(request.bodyInputStream.available(), 0);
- gHttpServAnother.registerPathHandler("/safebrowsing/update", function(request, response) {
- let body = NetUtil.readInputStreamToString(request.bodyInputStream,
- request.bodyInputStream.available());
+ // Not on the spec. Found in Chromium source code...
+ equal(request.getHeader("X-HTTP-Method-Override"), "POST");
+
+ // V4 update request uses GET.
+ equal(request.method, "GET");
- // Verify if the request is as expected.
- equal(body, TEST_TABLE_DATA_ANOTHER.tableName + ";\n");
+ // V4 append the base64 encoded request to the query string.
+ equal(request.queryString, gExpectedQueryV4);
- // Respond with no chunk control.
+ // Respond a V2 compatible content for now. In the future we can
+ // send a meaningful response to test Bug 1284178 to see if the
+ // update is successfully stored to database.
response.setHeader("Content-Type",
"application/vnd.google.safebrowsing-update", false);
response.setStatusLine(request.httpVersion, 200, "OK");
-
let content = "n:1000\n";
response.bodyOutputStream.write(content, content.length);
- gIsAnotherUpdated = true;
+ gIsV4Updated = true;
if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) {
// All tests are done!
run_next_test();
return;
}
do_print("Wait for all sever-involved tests to be done ...");
});
- gHttpServAnother.start(5555);
+ gHttpServV4.start(5555);
run_next_test();
}
// A trick to force updating tables. However, before calling this, we have to
// call disableAllUpdates() first to clean up the updateCheckers in listmanager.
function forceTableUpdate() {
prefBranch.setCharPref(PREF_NEXTUPDATETIME, "1");
+ prefBranch.setCharPref(PREF_NEXTUPDATETIME_V4, "1");
gListManager.maybeToggleUpdateChecking();
}
function disableAllUpdates() {
TEST_TABLE_DATA_LIST.forEach(t => gListManager.disableUpdate(t.tableName));
- gListManager.disableUpdate(TEST_TABLE_DATA_ANOTHER.tableName);
+ gListManager.disableUpdate(TEST_TABLE_DATA_V4.tableName);
}
// Since there's no public interface on listmanager to know the update success,
// we could only rely on the refresh of "nextupdatetime".
function waitForUpdateSuccess(callback) {
let nextupdatetime = parseInt(prefBranch.getCharPref(PREF_NEXTUPDATETIME));
do_print("nextupdatetime: " + nextupdatetime);
if (nextupdatetime !== 1) {
@@ -238,8 +261,16 @@ function waitForUpdateSuccess(callback)
function readFileToString(aFilename) {
let f = do_get_file(aFilename);
let stream = Cc["@mozilla.org/network/file-input-stream;1"]
.createInstance(Ci.nsIFileInputStream);
stream.init(f, -1, 0, 0);
let buf = NetUtil.readInputStreamToString(stream, stream.available());
return buf;
}
+
+function buildUpdateRequestV4InBase64() {
+
+ let request = urlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
+ [""],
+ 1);
+ return btoa(request);
+}