--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -5463,16 +5463,18 @@ pref("browser.safebrowsing.provider.goog
pref("browser.safebrowsing.provider.google4.lists", "goog-badbinurl-proto,goog-downloadwhite-proto,goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto,goog-harmful-proto,goog-passwordwhite-proto");
pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%&$httpMethod=POST");
pref("browser.safebrowsing.provider.google4.gethashURL", "https://safebrowsing.googleapis.com/v4/fullHashes:find?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%&$httpMethod=POST");
pref("browser.safebrowsing.provider.google4.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
pref("browser.safebrowsing.provider.google4.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.provider.google4.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.provider.google4.advisoryURL", "https://developers.google.com/safe-browsing/v4/advisory");
pref("browser.safebrowsing.provider.google4.advisoryName", "Google Safe Browsing");
+pref("browser.safebrowsing.provider.google4.dataSharingURL", "https://safebrowsing.googleapis.com/v4/threatHits?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%&$httpMethod=POST");
+pref("browser.safebrowsing.provider.google4.dataSharing.enabled", false);
pref("browser.safebrowsing.reportPhishURL", "https://%LOCALE%.phish-report.mozilla.com/?hl=%LOCALE%&url=");
// Mozilla Safe Browsing provider (for tracking protection and plugin blocking)
pref("browser.safebrowsing.provider.mozilla.pver", "2.2");
pref("browser.safebrowsing.provider.mozilla.lists", "base-track-digest256,mozstd-trackwhite-digest256,content-track-digest256,mozplugin-block-digest256,mozplugin2-block-digest256,block-flash-digest256,except-flash-digest256,allow-flashallow-digest256,except-flashallow-digest256,block-flashsubdoc-digest256,except-flashsubdoc-digest256,except-flashinfobar-digest256");
pref("browser.safebrowsing.provider.mozilla.updateURL", "https://shavar.services.mozilla.com/downloads?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2");
pref("browser.safebrowsing.provider.mozilla.gethashURL", "https://shavar.services.mozilla.com/gethash?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2");
--- a/netwerk/base/nsChannelClassifier.cpp
+++ b/netwerk/base/nsChannelClassifier.cpp
@@ -30,16 +30,18 @@
#include "nsISecurityEventSink.h"
#include "nsISupportsPriority.h"
#include "nsIURL.h"
#include "nsIWebProgressListener.h"
#include "nsNetUtil.h"
#include "nsPIDOMWindow.h"
#include "nsXULAppAPI.h"
#include "nsQueryObject.h"
+#include "nsIUrlClassifierDBService.h"
+#include "nsIURLFormatter.h"
#include "mozilla/ErrorNames.h"
#include "mozilla/Logging.h"
#include "mozilla/Preferences.h"
#include "mozilla/net/HttpBaseChannel.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/Unused.h"
@@ -862,17 +864,17 @@ nsChannelClassifier::SetBlockedContent(n
securityUI->GetState(&state);
if (aErrorCode == NS_ERROR_TRACKING_URI) {
doc->SetHasTrackingContentBlocked(true);
state |= nsIWebProgressListener::STATE_BLOCKED_TRACKING_CONTENT;
} else {
state |= nsIWebProgressListener::STATE_BLOCKED_UNSAFE_CONTENT;
}
- eventSink->OnSecurityChange(nullptr, state);
+ eventSink->OnSecurityChange(channel, state);
// Log a warning to the web console.
nsCOMPtr<nsIURI> uri;
channel->GetURI(getter_AddRefs(uri));
NS_ConvertUTF8toUTF16 spec(uri->GetSpecOrDefault());
const char16_t* params[] = { spec.get() };
const char* message = (aErrorCode == NS_ERROR_TRACKING_URI) ?
"TrackingUriBlocked" : "UnsafeUriBlocked";
@@ -1156,16 +1158,40 @@ nsChannelClassifier::IsTrackerWhiteliste
LOG(("nsChannelClassifier[%p]:IsTrackerWhitelisted whitelist disabled",
this));
return NS_ERROR_TRACKING_URI;
}
return uriClassifier->AsyncClassifyLocalWithTables(aWhiteListURI, trackingWhitelist, aCallback);
}
+nsresult
+nsChannelClassifier::SendThreatHitReport(nsIChannel *aChannel,
+ const nsACString& aProvider)
+{
+
+ nsAutoCString provider(aProvider);
+ nsPrintfCString reportEnablePref("browser.safebrowsing.provider.%s.dataSharing.enabled",
+ provider.get());
+ if (!Preferences::GetBool(reportEnablePref.get(), false)) {
+ return NS_OK;
+ }
+
+ nsCOMPtr<nsIURIClassifier> uriClassifier =
+ do_GetService(NS_URLCLASSIFIERDBSERVICE_CONTRACTID);
+ if (!uriClassifier) {
+ return NS_ERROR_UNEXPECTED;
+ }
+
+ nsresult rv = uriClassifier->SendThreatHitReport(mChannel);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ return NS_OK;
+}
+
NS_IMETHODIMP
nsChannelClassifier::OnClassifyComplete(nsresult aErrorCode,
const nsACString& aList,
const nsACString& aProvider,
const nsACString& aFullHash)
{
// Should only be called in the parent process.
MOZ_ASSERT(XRE_IsParentProcess());
@@ -1211,16 +1237,23 @@ nsChannelClassifier::OnClassifyCompleteI
}
// Channel will be cancelled (page element blocked) due to tracking
// protection or Safe Browsing.
// Do update the security state of the document and fire a security
// change event.
SetBlockedContent(mChannel, aErrorCode, aList, aProvider, aFullHash);
+ if (aErrorCode == NS_ERROR_MALWARE_URI ||
+ aErrorCode == NS_ERROR_PHISHING_URI ||
+ aErrorCode == NS_ERROR_UNWANTED_URI ||
+ aErrorCode == NS_ERROR_HARMFUL_URI) {
+ SendThreatHitReport(mChannel, aProvider);
+ }
+
mChannel->Cancel(aErrorCode);
}
LOG(("nsChannelClassifier[%p]: resuming channel %p from "
"OnClassifyComplete", this, mChannel.get()));
mChannel->Resume();
}
mChannel = nullptr;
--- a/netwerk/base/nsChannelClassifier.h
+++ b/netwerk/base/nsChannelClassifier.h
@@ -84,16 +84,18 @@ private:
// by ShouldEnableTrackingProtection().
nsresult ShouldEnableTrackingProtectionInternal(nsIChannel *aChannel,
bool aAnnotationsOnly,
bool *result);
bool AddonMayLoad(nsIChannel *aChannel, nsIURI *aUri);
void AddShutdownObserver();
void RemoveShutdownObserver();
+ nsresult SendThreatHitReport(nsIChannel *aChannel,
+ const nsACString& aProvider);
public:
// If we are blocking content, update the corresponding flag in the respective
// docshell and call nsISecurityEventSink::onSecurityChange.
static nsresult SetBlockedContent(nsIChannel *channel,
nsresult aErrorCode,
const nsACString& aList,
const nsACString& aProvider,
const nsACString& aFullHash);
--- a/netwerk/base/nsIURIClassifier.idl
+++ b/netwerk/base/nsIURIClassifier.idl
@@ -96,9 +96,18 @@ interface nsIURIClassifier : nsISupports
void asyncClassifyLocalWithTables(in nsIURI aURI,
in ACString aTables,
in nsIURIClassifierCallback aCallback);
/**
* Same as above, but returns a comma separated list of table names.
* This is an internal interface used only for testing purposes.
*/
ACString classifyLocal(in nsIURI aURI, in ACString aTables);
+
+ /**
+ * Report to the provider that a Safe Browsing warning was shown.
+ *
+ * @param aChannel
+ * Channel for which the URL matched something on the threat list.
+ */
+
+ void sendThreatHitReport(in nsIChannel aChannel);
};
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -5009,16 +5009,37 @@
"expires_in_version": "63",
"releaseChannelCollection": "opt-out",
"kind": "exponential",
"high": 86400000,
"n_buckets": 50,
"bug_numbers": [1338082],
"description": "Negative cache duration (ms) received in fullhash response from any v4 provider"
},
+ "URLCLASSIFIER_THREATHIT_NETWORK_ERROR": {
+ "record_in_processes": ["main", "content"],
+ "alert_emails": ["tnguyen@mozilla.com, safebrowsing-telemetry@mozilla.org"],
+ "expires_in_version": "never",
+ "releaseChannelCollection": "opt-out",
+ "kind": "enumerated",
+ "n_values": 30,
+ "bug_numbers": [1351147],
+ "description": "Whether or not an error was encountered while sending a Safe Browsing ThreatHit report. (0=sucess, 1=unknown error, 2=already connected, 3=not connected, 4=connection refused,5=net timeout, 6=offline, 7=port access not allowed, 8=net reset, 9=net interrupt, 10=proxy connection refused, 11=partial transfer, 12=inadequate security, 13=unknown host, 14=dns lookup queue full, 15=unknown proxy host)"
+ },
+ "URLCLASSIFIER_THREATHIT_REMOTE_STATUS": {
+ "record_in_processes": ["main", "content"],
+ "alert_emails": ["tnguyen@mozilla.com, safebrowsing-telemetry@mozilla.org"],
+ "expires_in_version": "never",
+ "releaseChannelCollection": "opt-out",
+ "kind": "enumerated",
+ "n_values": 16,
+ "bug_numbers": [1351147],
+ "description": "Server HTTP status code from Safe Browsing ThreatHit report. (0=1xx, 1=200, 2=2xx, 3=204, 4=3xx, 5=400, 6=4xx, 7=403, 8=404, 9=408, 10=413, 11=5xx, 12=502|504|511, 13=503, 14=505, 15=Other)"
+ },
+
"CSP_DOCUMENTS_COUNT": {
"record_in_processes": ["main", "content"],
"alert_emails": ["seceng-telemetry@mozilla.com"],
"bug_numbers": [1252829],
"expires_in_version": "60",
"kind": "count",
"description": "Number of unique pages that contain a CSP"
},
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/UrlClassifierTelemetryUtils.cpp
@@ -0,0 +1,146 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "UrlClassifierTelemetryUtils.h"
+#include "mozilla/Assertions.h"
+
+namespace mozilla {
+namespace safebrowsing {
+
+uint8_t
+NetworkErrorToBucket(nsresult rv)
+{
+ switch(rv) {
+ // Connection errors
+ case NS_ERROR_ALREADY_CONNECTED: return 2;
+ case NS_ERROR_NOT_CONNECTED: return 3;
+ case NS_ERROR_CONNECTION_REFUSED: return 4;
+ case NS_ERROR_NET_TIMEOUT: return 5;
+ case NS_ERROR_OFFLINE: return 6;
+ case NS_ERROR_PORT_ACCESS_NOT_ALLOWED: return 7;
+ case NS_ERROR_NET_RESET: return 8;
+ case NS_ERROR_NET_INTERRUPT: return 9;
+ case NS_ERROR_PROXY_CONNECTION_REFUSED: return 10;
+ case NS_ERROR_NET_PARTIAL_TRANSFER: return 11;
+ case NS_ERROR_NET_INADEQUATE_SECURITY: return 12;
+ // DNS errors
+ case NS_ERROR_UNKNOWN_HOST: return 13;
+ case NS_ERROR_DNS_LOOKUP_QUEUE_FULL: return 14;
+ case NS_ERROR_UNKNOWN_PROXY_HOST: return 15;
+ // Others
+ default: return 1;
+ }
+}
+
+uint32_t
+HTTPStatusToBucket(uint32_t status)
+{
+ uint32_t statusBucket;
+ switch (status) {
+ case 100:
+ case 101:
+ // Unexpected 1xx return code
+ statusBucket = 0;
+ break;
+ case 200:
+ // OK - Data is available in the HTTP response body.
+ statusBucket = 1;
+ break;
+ case 201:
+ case 202:
+ case 203:
+ case 205:
+ case 206:
+ // Unexpected 2xx return code
+ statusBucket = 2;
+ break;
+ case 204:
+ // No Content
+ statusBucket = 3;
+ break;
+ case 300:
+ case 301:
+ case 302:
+ case 303:
+ case 304:
+ case 305:
+ case 307:
+ case 308:
+ // Unexpected 3xx return code
+ statusBucket = 4;
+ break;
+ case 400:
+ // Bad Request - The HTTP request was not correctly formed.
+ // The client did not provide all required CGI parameters.
+ statusBucket = 5;
+ break;
+ case 401:
+ case 402:
+ case 405:
+ case 406:
+ case 407:
+ case 409:
+ case 410:
+ case 411:
+ case 412:
+ case 414:
+ case 415:
+ case 416:
+ case 417:
+ case 421:
+ case 426:
+ case 428:
+ case 429:
+ case 431:
+ case 451:
+ // Unexpected 4xx return code
+ statusBucket = 6;
+ break;
+ case 403:
+ // Forbidden - The client id is invalid.
+ statusBucket = 7;
+ break;
+ case 404:
+ // Not Found
+ statusBucket = 8;
+ break;
+ case 408:
+ // Request Timeout
+ statusBucket = 9;
+ break;
+ case 413:
+ // Request Entity Too Large - Bug 1150334
+ statusBucket = 10;
+ break;
+ case 500:
+ case 501:
+ case 510:
+ // Unexpected 5xx return code
+ statusBucket = 11;
+ break;
+ case 502:
+ case 504:
+ case 511:
+ // Local network errors, we'll ignore these.
+ statusBucket = 12;
+ break;
+ case 503:
+ // Service Unavailable - The server cannot handle the request.
+ // Clients MUST follow the backoff behavior specified in the
+ // Request Frequency section.
+ statusBucket = 13;
+ break;
+ case 505:
+ // HTTP Version Not Supported - The server CANNOT handle the requested
+ // protocol major version.
+ statusBucket = 14;
+ break;
+ default:
+ statusBucket = 15;
+ };
+ return statusBucket;
+}
+
+} // namespace safebrowsing
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/UrlClassifierTelemetryUtils.h
@@ -0,0 +1,32 @@
+//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef UrlClassifierTelemetryUtils_h__
+#define UrlClassifierTelemetryUtils_h__
+
+#include "mozilla/TypedEnumBits.h"
+
+namespace mozilla {
+namespace safebrowsing {
+
+// We might need to expand the bucket here if telemetry shows lots of errors
+// are neither connection errors nor DNS errors.
+uint8_t
+NetworkErrorToBucket(nsresult rv);
+
+// Map the HTTP response code to a Telemetry bucket
+uint32_t
+HTTPStatusToBucket(uint32_t status);
+
+enum UpdateTimeout {
+ eNoTimeout = 0,
+ eResponseTimeout = 1,
+ eDownloadTimeout = 2,
+};
+
+} // namespace safebrowsing
+} // namespace mozilla
+
+#endif //UrlClassifierTelemetryUtils_h__
--- a/toolkit/components/url-classifier/moz.build
+++ b/toolkit/components/url-classifier/moz.build
@@ -33,16 +33,17 @@ UNIFIED_SOURCES += [
'nsCheckSummedOutputStream.cpp',
'nsUrlClassifierDBService.cpp',
'nsUrlClassifierInfo.cpp',
'nsUrlClassifierProxies.cpp',
'nsUrlClassifierUtils.cpp',
'protobuf/safebrowsing.pb.cc',
'ProtocolParser.cpp',
'RiceDeltaDecoder.cpp',
+ 'UrlClassifierTelemetryUtils.cpp',
]
# define conflicting LOG() macros
SOURCES += [
'nsUrlClassifierPrefixSet.cpp',
'nsUrlClassifierStreamUpdater.cpp',
'VariableLengthPrefixSet.cpp',
]
--- a/toolkit/components/url-classifier/nsIUrlClassifierUtils.idl
+++ b/toolkit/components/url-classifier/nsIUrlClassifierUtils.idl
@@ -3,16 +3,17 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsISupports.idl"
/**
* Some utility methods used by the url classifier.
*/
interface nsIURI;
+interface nsIChannel;
/**
* Interface for parseFindFullHashResponseV4 callback
*/
[scriptable, uuid(fbb9684a-a0aa-11e6-88b0-08606e456b8a)]
interface nsIUrlClassifierParseFindFullHashCallback : nsISupports {
/**
* Callback when a match is found in full hash response. This callback may be
@@ -130,16 +131,29 @@ interface nsIUrlClassifierUtils : nsISup
*/
ACString makeFindFullHashRequestV4([array, size_is(aListCount)] in string aListNames,
[array, size_is(aListCount)] in string aListStatesBase64,
[array, size_is(aPrefixCount)] in string aPrefixes,
in uint32_t aListCount,
in uint32_t aPrefixCount);
/**
+ * Make ThreatHit report request body.
+ *
+ * @param aChannel channel which encountered the threat.
+ * @param aListName listname represented in string.
+ * @param aHashBase64 hash-based hit represented in base64.
+ *
+ * @returns A base64 encoded string.
+ */
+ ACString makeThreatHitReport(in nsIChannel aChannel,
+ in ACString aListName,
+ in ACString aHashBase64);
+
+ /**
* Parse V4 FindFullHash response.
*
* @param aResponse Byte stream from the server.
* @param aCallback The callback function on each complete hash parsed.
* Can be called multiple times in one parsing.
*/
void parseFindFullHashResponseV4(in ACString aResponse,
in nsIUrlClassifierParseFindFullHashCallback aCallback);
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -46,16 +46,23 @@
#include "Classifier.h"
#include "ProtocolParser.h"
#include "nsContentUtils.h"
#include "mozilla/dom/ContentChild.h"
#include "mozilla/dom/PermissionMessageUtils.h"
#include "mozilla/dom/URLClassifierChild.h"
#include "mozilla/ipc/URIUtils.h"
#include "nsProxyRelease.h"
+#include "UrlClassifierTelemetryUtils.h"
+#include "nsIURLFormatter.h"
+#include "nsIUploadChannel.h"
+#include "nsStringStream.h"
+#include "nsNetUtil.h"
+#include "nsToolkitCompsCID.h"
+#include "nsIClassifiedChannel.h"
namespace mozilla {
namespace safebrowsing {
nsresult
TablesToResponse(const nsACString& tables)
{
if (tables.IsEmpty()) {
@@ -1917,16 +1924,206 @@ nsUrlClassifierDBService::ClassifyLocalW
rv = mWorkerProxy->DoLocalLookup(key, aTables, results);
if (NS_SUCCEEDED(rv)) {
rv = ProcessLookupResults(results, aTableResults);
NS_ENSURE_SUCCESS(rv, rv);
}
return NS_OK;
}
+class ThreatHitReportListener final
+ : public nsIStreamListener
+{
+public:
+ NS_DECL_ISUPPORTS
+ NS_DECL_NSIREQUESTOBSERVER
+ NS_DECL_NSISTREAMLISTENER
+
+ ThreatHitReportListener() = default;
+
+private:
+ ~ThreatHitReportListener() = default;
+};
+
+NS_IMPL_ISUPPORTS(ThreatHitReportListener, nsIStreamListener, nsIRequestObserver)
+
+NS_IMETHODIMP
+ThreatHitReportListener::OnStartRequest(nsIRequest* aRequest, nsISupports* aContext)
+{
+ nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
+ if (httpChannel) {
+ nsresult rv;
+ nsresult status = NS_OK;
+ rv = httpChannel->GetStatus(&status);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ uint8_t netErrCode = NS_FAILED(status) ?
+ mozilla::safebrowsing::NetworkErrorToBucket(status) : 0;
+ mozilla::Telemetry::Accumulate(
+ mozilla::Telemetry::URLCLASSIFIER_THREATHIT_NETWORK_ERROR, netErrCode);
+
+ uint32_t requestStatus;
+ rv = httpChannel->GetResponseStatus(&requestStatus);
+ NS_ENSURE_SUCCESS(rv, rv);
+ mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_THREATHIT_REMOTE_STATUS,
+ mozilla::safebrowsing::HTTPStatusToBucket(requestStatus));
+ if (LOG_ENABLED()) {
+ nsAutoCString errorName, spec;
+ mozilla::GetErrorName(status, errorName);
+ nsCOMPtr<nsIURI> uri;
+ rv = httpChannel->GetURI(getter_AddRefs(uri));
+ if (NS_SUCCEEDED(rv) && uri) {
+ uri->GetAsciiSpec(spec);
+ }
+
+ nsCOMPtr<nsIURLFormatter> urlFormatter =
+ do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
+
+ // Trim sensitive log data
+ nsString trimmedSpec;
+ rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmedSpec);
+ if (NS_SUCCEEDED(rv)) {
+ LOG(("ThreatHitReportListener::OnStartRequest "
+ "(status=%s, uri=%s, this=%p)", errorName.get(),
+ NS_ConvertUTF16toUTF8(trimmedSpec).get(), this));
+
+ }
+ }
+
+ LOG(("ThreatHit report response %d %d", requestStatus, netErrCode));
+ }
+
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+ThreatHitReportListener::OnDataAvailable(nsIRequest* aRequest,
+ nsISupports* aContext,
+ nsIInputStream* aInputStream,
+ uint64_t aOffset,
+ uint32_t aCount)
+{
+ return NS_OK;
+}
+NS_IMETHODIMP
+ThreatHitReportListener::OnStopRequest(nsIRequest* aRequest, nsISupports* aContext,
+ nsresult aStatus)
+{
+ return aStatus;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::SendThreatHitReport(nsIChannel *aChannel)
+{
+ NS_ENSURE_ARG_POINTER(aChannel);
+ nsresult rv;
+
+ nsCOMPtr<nsIClassifiedChannel> classifiedChannel =
+ do_QueryInterface(aChannel, &rv);
+ NS_ENSURE_SUCCESS(rv, rv);
+ if (!classifiedChannel) {
+ return NS_ERROR_FAILURE;
+ }
+
+ nsAutoCString provider;
+ rv = classifiedChannel->GetMatchedProvider(provider);
+ if (NS_FAILED(rv) || provider.IsEmpty()) {
+ return NS_ERROR_FAILURE;
+ }
+
+ nsAutoCString listName;
+ rv = classifiedChannel->GetMatchedList(listName);
+ if (NS_FAILED(rv) || listName.IsEmpty()) {
+ return NS_ERROR_FAILURE;
+ }
+
+ nsAutoCString fullHash;
+ rv = classifiedChannel->GetMatchedFullHash(fullHash);
+ if (NS_FAILED(rv) || fullHash.IsEmpty()) {
+ return NS_ERROR_FAILURE;
+ }
+
+ nsPrintfCString reportUrlPref("browser.safebrowsing.provider.%s.dataSharingURL",
+ provider.get());
+
+ nsCOMPtr<nsIURLFormatter> formatter(
+ do_GetService("@mozilla.org/toolkit/URLFormatterService;1"));
+ if (!formatter) {
+ return NS_ERROR_UNEXPECTED;
+ }
+
+ nsString urlStr;
+ rv = formatter->FormatURLPref(NS_ConvertUTF8toUTF16(reportUrlPref), urlStr);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ if (urlStr.IsEmpty() || NS_LITERAL_STRING("about:blank").Equals(urlStr)) {
+ LOG(("%s is missing a ThreatHit data reporting URL.", provider.get()));
+ return NS_OK;
+ }
+
+ nsCOMPtr<nsIUrlClassifierUtils> utilsService =
+ do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
+ if (!utilsService) {
+ return NS_ERROR_FAILURE;
+ }
+
+ nsAutoCString reportBody;
+ rv = utilsService->MakeThreatHitReport(aChannel, listName, fullHash, reportBody);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ nsAutoCString reportUriStr = NS_ConvertUTF16toUTF8(urlStr);
+ reportUriStr.Append("&$req=");
+ reportUriStr.Append(reportBody);
+
+ nsCOMPtr<nsIURI> reportURI;
+ rv = NS_NewURI(getter_AddRefs(reportURI), reportUriStr);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ uint32_t loadFlags = nsIChannel::INHIBIT_CACHING |
+ nsIChannel::LOAD_BYPASS_CACHE;
+
+ nsCOMPtr<nsIChannel> reportChannel;
+ rv = NS_NewChannel(getter_AddRefs(reportChannel),
+ reportURI,
+ nsContentUtils::GetSystemPrincipal(),
+ nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
+ nsIContentPolicy::TYPE_OTHER,
+ nullptr, // aLoadGroup
+ nullptr,
+ loadFlags);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ // Safe Browsing has a separate cookie jar
+ nsCOMPtr<nsILoadInfo> loadInfo = reportChannel->GetLoadInfo();
+ mozilla::OriginAttributes attrs;
+ attrs.mFirstPartyDomain.AssignLiteral(NECKO_SAFEBROWSING_FIRST_PARTY_DOMAIN);
+ if (loadInfo) {
+ loadInfo->SetOriginAttributes(attrs);
+ }
+
+ nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(reportChannel));
+ NS_ENSURE_TRUE(httpChannel, rv);
+
+ rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST"));
+ NS_ENSURE_SUCCESS(rv, rv);
+ // Disable keepalive.
+ rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("Connection"), NS_LITERAL_CSTRING("close"), false);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ RefPtr<ThreatHitReportListener> listener = new ThreatHitReportListener();
+ rv = reportChannel->AsyncOpen2(listener);
+ if (NS_FAILED(rv)) {
+ LOG(("Failure to send Safe Browsing ThreatHit report"));
+ return rv;
+ }
+
+ return NS_OK;
+}
+
+
NS_IMETHODIMP
nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal,
const nsACString& tables,
nsIUrlClassifierCallback* c)
{
NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
bool dummy;
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp
@@ -20,16 +20,17 @@
#include "mozilla/ErrorNames.h"
#include "mozilla/Logging.h"
#include "nsIInterfaceRequestor.h"
#include "mozilla/LoadContext.h"
#include "mozilla/Telemetry.h"
#include "nsContentUtils.h"
#include "nsIURLFormatter.h"
#include "Classifier.h"
+#include "UrlClassifierTelemetryUtils.h"
using namespace mozilla::safebrowsing;
using namespace mozilla;
#define DEFAULT_RESPONSE_TIMEOUT_MS 15 * 1000
#define DEFAULT_TIMEOUT_MS 60 * 1000
static_assert(DEFAULT_TIMEOUT_MS > DEFAULT_RESPONSE_TIMEOUT_MS,
"General timeout must be greater than reponse timeout");
@@ -575,151 +576,16 @@ nsUrlClassifierStreamUpdater::AddRequest
NS_ENSURE_SUCCESS(rv, rv);
rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST"));
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
-// We might need to expand the bucket here if telemetry shows lots of errors
-// are neither connection errors nor DNS errors.
-static uint8_t NetworkErrorToBucket(nsresult rv)
-{
- switch(rv) {
- // Connection errors
- case NS_ERROR_ALREADY_CONNECTED: return 2;
- case NS_ERROR_NOT_CONNECTED: return 3;
- case NS_ERROR_CONNECTION_REFUSED: return 4;
- case NS_ERROR_NET_TIMEOUT: return 5;
- case NS_ERROR_OFFLINE: return 6;
- case NS_ERROR_PORT_ACCESS_NOT_ALLOWED: return 7;
- case NS_ERROR_NET_RESET: return 8;
- case NS_ERROR_NET_INTERRUPT: return 9;
- case NS_ERROR_PROXY_CONNECTION_REFUSED: return 10;
- case NS_ERROR_NET_PARTIAL_TRANSFER: return 11;
- case NS_ERROR_NET_INADEQUATE_SECURITY: return 12;
- // DNS errors
- case NS_ERROR_UNKNOWN_HOST: return 13;
- case NS_ERROR_DNS_LOOKUP_QUEUE_FULL: return 14;
- case NS_ERROR_UNKNOWN_PROXY_HOST: return 15;
- // Others
- default: return 1;
- }
-}
-
-// Map the HTTP response code to a Telemetry bucket
-static uint32_t HTTPStatusToBucket(uint32_t status)
-{
- uint32_t statusBucket;
- switch (status) {
- case 100:
- case 101:
- // Unexpected 1xx return code
- statusBucket = 0;
- break;
- case 200:
- // OK - Data is available in the HTTP response body.
- statusBucket = 1;
- break;
- case 201:
- case 202:
- case 203:
- case 205:
- case 206:
- // Unexpected 2xx return code
- statusBucket = 2;
- break;
- case 204:
- // No Content
- statusBucket = 3;
- break;
- case 300:
- case 301:
- case 302:
- case 303:
- case 304:
- case 305:
- case 307:
- case 308:
- // Unexpected 3xx return code
- statusBucket = 4;
- break;
- case 400:
- // Bad Request - The HTTP request was not correctly formed.
- // The client did not provide all required CGI parameters.
- statusBucket = 5;
- break;
- case 401:
- case 402:
- case 405:
- case 406:
- case 407:
- case 409:
- case 410:
- case 411:
- case 412:
- case 414:
- case 415:
- case 416:
- case 417:
- case 421:
- case 426:
- case 428:
- case 429:
- case 431:
- case 451:
- // Unexpected 4xx return code
- statusBucket = 6;
- break;
- case 403:
- // Forbidden - The client id is invalid.
- statusBucket = 7;
- break;
- case 404:
- // Not Found
- statusBucket = 8;
- break;
- case 408:
- // Request Timeout
- statusBucket = 9;
- break;
- case 413:
- // Request Entity Too Large - Bug 1150334
- statusBucket = 10;
- break;
- case 500:
- case 501:
- case 510:
- // Unexpected 5xx return code
- statusBucket = 11;
- break;
- case 502:
- case 504:
- case 511:
- // Local network errors, we'll ignore these.
- statusBucket = 12;
- break;
- case 503:
- // Service Unavailable - The server cannot handle the request.
- // Clients MUST follow the backoff behavior specified in the
- // Request Frequency section.
- statusBucket = 13;
- break;
- case 505:
- // HTTP Version Not Supported - The server CANNOT handle the requested
- // protocol major version.
- statusBucket = 14;
- break;
- default:
- statusBucket = 15;
- };
- return statusBucket;
-}
-
///////////////////////////////////////////////////////////////////////////////
// nsIStreamListenerObserver implementation
NS_IMETHODIMP
nsUrlClassifierStreamUpdater::OnStartRequest(nsIRequest *request,
nsISupports* context)
{
nsresult rv;
--- a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h
+++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h
@@ -66,22 +66,16 @@ private:
bool aIsPostRequest,
const nsACString &aTable);
// Fetches the next table, from mPendingUpdates.
nsresult FetchNext();
// Fetches the next request, from mPendingRequests
nsresult FetchNextRequest();
- enum UpdateTimeout {
- eNoTimeout = 0,
- eResponseTimeout = 1,
- eDownloadTimeout = 2,
- };
-
bool mIsUpdating;
bool mInitialized;
bool mDownloadError;
bool mBeganStream;
nsCString mDownloadErrorStatusStr;
// Note that mStreamTable is only used by v2, it is empty for v4 update.
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
@@ -1,23 +1,28 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsEscape.h"
#include "nsString.h"
#include "nsIURI.h"
+#include "nsIURL.h"
#include "nsUrlClassifierUtils.h"
#include "nsTArray.h"
#include "nsReadableUtils.h"
#include "plbase64.h"
#include "nsPrintfCString.h"
#include "safebrowsing.pb.h"
#include "mozilla/Sprintf.h"
#include "mozilla/Mutex.h"
+#include "nsIRedirectHistoryEntry.h"
+#include "nsIHttpChannelInternal.h"
+#include "mozIThirdPartyUtil.h"
+#include "nsIDocShell.h"
#define DEFAULT_PROTOCOL_VERSION "2.2"
static char int_to_hex_digit(int32_t i)
{
NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit");
return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A')));
}
@@ -461,16 +466,258 @@ nsUrlClassifierUtils::MakeFindFullHashRe
out);
NS_ENSURE_SUCCESS(rv, rv);
aRequest = out;
return NS_OK;
}
+// Remove ref, query, userpass, anypart which may contain sensitive data
+static nsresult
+GetSpecWithoutSensitiveData(nsIURI* aUri, nsACString &aSpec)
+{
+ if (NS_WARN_IF(!aUri)) {
+ return NS_ERROR_INVALID_ARG;
+ }
+
+ nsCOMPtr<nsIURI> clone;
+ // Clone to make the uri mutable
+ nsresult rv = aUri->CloneIgnoringRef(getter_AddRefs(clone));
+ nsCOMPtr<nsIURL> url(do_QueryInterface(clone));
+ if (url) {
+ rv = url->SetQuery(EmptyCString());
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = url->SetRef(EmptyCString());
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = url->SetUserPass(EmptyCString());
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = url->GetAsciiSpec(aSpec);
+ NS_ENSURE_SUCCESS(rv, rv);
+ }
+ return NS_OK;
+}
+
+static nsresult
+AddThreatSourceFromChannel(ThreatHit& aHit, nsIChannel *aChannel,
+ ThreatHit_ThreatSourceType aType)
+{
+ if (NS_WARN_IF(!aChannel)) {
+ return NS_ERROR_INVALID_ARG;
+ }
+
+ nsresult rv;
+
+ auto matchingSource = aHit.add_resources();
+ matchingSource->set_type(aType);
+
+ nsCOMPtr<nsIURI> uri;
+ rv = aChannel->GetURI(getter_AddRefs(uri));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ nsCString spec;
+ rv = GetSpecWithoutSensitiveData(uri, spec);
+ NS_ENSURE_SUCCESS(rv, rv);
+ matchingSource->set_url(spec.get());
+
+ nsCOMPtr<nsIHttpChannel> httpChannel =
+ do_QueryInterface(aChannel);
+ if (httpChannel) {
+ nsCOMPtr<nsIURI> referrer;
+ rv = httpChannel->GetReferrer(getter_AddRefs(referrer));
+ if (NS_SUCCEEDED(rv) && referrer) {
+ nsCString referrerSpec;
+ rv = GetSpecWithoutSensitiveData(referrer, referrerSpec);
+ NS_ENSURE_SUCCESS(rv, rv);
+ matchingSource->set_referrer(referrerSpec.get());
+ }
+ }
+
+ nsCOMPtr<nsIHttpChannelInternal> httpChannelInternal =
+ do_QueryInterface(aChannel);
+ if (httpChannelInternal) {
+ nsCString remoteIp;
+ rv = httpChannelInternal->GetRemoteAddress(remoteIp);
+ if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) {
+ matchingSource->set_remote_ip(remoteIp.get());
+ }
+ }
+ return NS_OK;
+}
+static nsresult
+AddThreatSourceFromRedirectEntry(ThreatHit& aHit,
+ nsIRedirectHistoryEntry *aRedirectEntry,
+ ThreatHit_ThreatSourceType aType)
+{
+ if (NS_WARN_IF(!aRedirectEntry)) {
+ return NS_ERROR_INVALID_ARG;
+ }
+
+ nsresult rv;
+
+ nsCOMPtr<nsIPrincipal> principal;
+ rv = aRedirectEntry->GetPrincipal(getter_AddRefs(principal));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ nsCOMPtr<nsIURI> uri;
+ rv = principal->GetURI(getter_AddRefs(uri));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ nsCString spec;
+ rv = GetSpecWithoutSensitiveData(uri, spec);
+ NS_ENSURE_SUCCESS(rv, rv);
+ auto source = aHit.add_resources();
+ source->set_url(spec.get());
+ source->set_type(aType);
+
+ nsCOMPtr<nsIURI> referrer;
+ rv = aRedirectEntry->GetReferrerURI(getter_AddRefs(referrer));
+ if (NS_SUCCEEDED(rv) && referrer) {
+ nsCString referrerSpec;
+ rv = GetSpecWithoutSensitiveData(referrer, referrerSpec);
+ NS_ENSURE_SUCCESS(rv, rv);
+ source->set_referrer(referrerSpec.get());
+ }
+
+ nsCString remoteIp;
+ rv = aRedirectEntry->GetRemoteAddress(remoteIp);
+ if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) {
+ source->set_remote_ip(remoteIp.get());
+ }
+ return NS_OK;
+}
+
+// Add top level tab url and redirect threatsources to threatHit message
+static nsresult
+AddTabThreatSources(ThreatHit& aHit, nsIChannel *aChannel)
+{
+ if (NS_WARN_IF(!aChannel)) {
+ return NS_ERROR_INVALID_ARG;
+ }
+
+ nsresult rv;
+ nsCOMPtr<mozIDOMWindowProxy> win;
+ nsCOMPtr<mozIThirdPartyUtil> thirdPartyUtil =
+ do_GetService(THIRDPARTYUTIL_CONTRACTID, &rv);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = thirdPartyUtil->GetTopWindowForChannel(aChannel, getter_AddRefs(win));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ auto* pwin = nsPIDOMWindowOuter::From(win);
+ nsCOMPtr<nsIDocShell> docShell = pwin->GetDocShell();
+ if (!docShell) {
+ return NS_OK;
+ }
+
+ nsCOMPtr<nsIChannel> topChannel;
+ docShell->GetCurrentDocumentChannel(getter_AddRefs(topChannel));
+ if (!topChannel) {
+ return NS_OK;
+ }
+
+ nsCOMPtr<nsIURI> uri;
+ rv = aChannel->GetURI(getter_AddRefs(uri));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ nsCOMPtr<nsIURI> topUri;
+ rv = topChannel->GetURI(getter_AddRefs(topUri));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ bool isTopUri = false;
+ rv = topUri->Equals(uri, &isTopUri);
+ if (NS_SUCCEEDED(rv) && !isTopUri) {
+ nsCOMPtr<nsILoadInfo> loadInfo = aChannel->GetLoadInfo();
+ if (loadInfo && loadInfo->RedirectChain().Length()) {
+ AddThreatSourceFromRedirectEntry(aHit, loadInfo->RedirectChain()[0],
+ ThreatHit_ThreatSourceType_TAB_RESOURCE);
+ }
+ }
+
+ // Set top level tab_url threatshource
+ rv = AddThreatSourceFromChannel(aHit, topChannel,
+ ThreatHit_ThreatSourceType_TAB_URL);
+ Unused << NS_WARN_IF(NS_FAILED(rv));
+
+ // Set tab_redirect threatshources if there's any
+ nsCOMPtr<nsILoadInfo> topLoadInfo = topChannel->GetLoadInfo();
+ if (!topLoadInfo) {
+ return NS_OK;
+ }
+
+ nsIRedirectHistoryEntry* redirectEntry;
+ size_t length = topLoadInfo->RedirectChain().Length();
+ for (size_t i = 0; i < length; i++) {
+ redirectEntry = topLoadInfo->RedirectChain()[i];
+ AddThreatSourceFromRedirectEntry(aHit, redirectEntry,
+ ThreatHit_ThreatSourceType_TAB_REDIRECT);
+ }
+
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierUtils::MakeThreatHitReport(nsIChannel *aChannel,
+ const nsACString& aListName,
+ const nsACString& aHashBase64,
+ nsACString &aRequest)
+{
+ if (NS_WARN_IF(aListName.IsEmpty()) ||
+ NS_WARN_IF(aHashBase64.IsEmpty()) ||
+ NS_WARN_IF(!aChannel)) {
+ return NS_ERROR_INVALID_ARG;
+ }
+
+ ThreatHit hit;
+ nsresult rv;
+
+ uint32_t threatType;
+ rv = ConvertListNameToThreatType(aListName, &threatType);
+ NS_ENSURE_SUCCESS(rv, rv);
+ hit.set_threat_type(static_cast<ThreatType>(threatType));
+
+ hit.set_platform_type(GetPlatformType());
+
+ nsCString hash;
+ rv = Base64Decode(aHashBase64, hash);
+ if (NS_FAILED(rv) || hash.Length() != COMPLETE_SIZE) {
+ return NS_ERROR_FAILURE;
+ }
+
+ auto threatEntry = hit.mutable_entry();
+ threatEntry->set_hash(hash.get(), hash.Length());
+
+ // Set matching source
+ rv = AddThreatSourceFromChannel(hit, aChannel,
+ ThreatHit_ThreatSourceType_MATCHING_URL);
+ Unused << NS_WARN_IF(NS_FAILED(rv));
+ // Set tab url, tab resource url and redirect sources
+ rv = AddTabThreatSources(hit, aChannel);
+ Unused << NS_WARN_IF(NS_FAILED(rv));
+
+ hit.set_allocated_client_info(CreateClientInfo());
+
+ std::string s;
+ hit.SerializeToString(&s);
+
+ nsCString out;
+ rv = Base64URLEncode(s.size(),
+ reinterpret_cast<const uint8_t*>(s.c_str()),
+ Base64URLEncodePaddingPolicy::Include,
+ out);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ aRequest = out;
+
+ return NS_OK;
+}
+
static uint32_t
DurationToMs(const Duration& aDuration)
{
// Seconds precision is good enough. Ignore nanoseconds like Chrome does.
return aDuration.seconds() * 1000;
}
NS_IMETHODIMP
--- a/toolkit/components/url-classifier/tests/mochitest/chrome.ini
+++ b/toolkit/components/url-classifier/tests/mochitest/chrome.ini
@@ -6,16 +6,17 @@ support-files =
classifiedAnnotatedPBFrame.html
trackingRequest.html
bug_1281083.html
report.sjs
gethash.sjs
classifierCommon.js
classifierHelper.js
head.js
+ threathit.sjs
[test_lookup_system_principal.html]
[test_classified_annotations.html]
tags = trackingprotection
skip-if = os == 'linux' && asan # Bug 1202548
[test_allowlisted_annotations.html]
tags = trackingprotection
[test_privatebrowsing_trackingprotection.html]
@@ -26,8 +27,9 @@ tags = trackingprotection
tags = trackingprotection
[test_safebrowsing_bug1272239.html]
[test_donottrack.html]
[test_classifier_changetablepref.html]
[test_classifier_changetablepref_bug1395411.html]
[test_reporturl.html]
[test_trackingprotection_bug1312515.html]
[test_advisory_link.html]
+[test_threathit_report.html]
--- a/toolkit/components/url-classifier/tests/mochitest/head.js
+++ b/toolkit/components/url-classifier/tests/mochitest/head.js
@@ -29,8 +29,21 @@ function hash(str) {
.createInstance(SpecialPowers.Ci.nsICryptoHash);
var data = bytesFromString(str);
hasher.init(hasher.SHA256);
hasher.update(data, data.length);
return hasher.finish(true);
}
+
+var pushPrefs = (...p) => SpecialPowers.pushPrefEnv({set: p});
+
+function whenDelayedStartupFinished(aWindow, aCallback) {
+ Services.obs.addObserver(function observer(aSubject, aTopic) {
+ if (aWindow == aSubject) {
+ Services.obs.removeObserver(observer, aTopic);
+ setTimeout(aCallback, 0);
+ }
+ }, "browser-delayed-startup-finished");
+}
+
+
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/test_threathit_report.html
@@ -0,0 +1,252 @@
+<!DOCTYPE HTML>
+<!-- Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ -->
+<html>
+<head>
+ <title>Test threathit repoty </title>
+ <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+ <script type="text/javascript" src="classifierHelper.js"></script>
+ <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css">
+</head>
+
+<body>
+<p id="display"></p>
+<div id="content" style="display: none">
+</div>
+<pre id="test">
+
+<script src="head.js"></script>
+<script class="testbody" type="text/javascript">
+/* import-globals-from classifierHelper.js */
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+const Cc = Components.classes;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://testing-common/BrowserTestUtils.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor)
+ .getInterface(Ci.nsIWebNavigation)
+ .QueryInterface(Ci.nsIDocShellTreeItem)
+ .rootTreeItem
+ .QueryInterface(Ci.nsIInterfaceRequestor)
+ .getInterface(Ci.nsIDOMWindow);
+
+var listmanager = Cc["@mozilla.org/url-classifier/listmanager;1"].
+ getService(Ci.nsIUrlListManager);
+const SJS = "mochi.test:8888/chrome/toolkit/components/url-classifier/tests/mochitest/threathit.sjs";
+
+function hash(str) {
+ function bytesFromString(str1) {
+ let converter =
+ Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+ .createInstance(Ci.nsIScriptableUnicodeConverter);
+ converter.charset = "UTF-8";
+ return converter.convertToByteArray(str1);
+ }
+
+ let hasher = Cc["@mozilla.org/security/hash;1"]
+ .createInstance(Ci.nsICryptoHash);
+
+ let data = bytesFromString(str);
+ hasher.init(hasher.SHA256);
+ hasher.update(data, data.length);
+
+ return hasher.finish(false);
+}
+
+var testDatas = [
+ { url: "itisaphishingsite1.org/phishing.html",
+ list: "test-phish-proto",
+ provider: "test",
+ // The base64 of binary protobuf representation of response:
+ //
+ // [
+ // {
+ // 'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
+ // 'response_type': 2, // FULL_UPDATE
+ // 'new_client_state': 'sta\x0te', // NEW_CLIENT_STATE
+ // 'additions': { 'compression_type': RAW,
+ // 'prefix_size': 1,
+ // 'raw_hashes': "xxxx"} // hash prefix of url itisaphishingsite.org/phishing.html
+ // 'minimumWaitDuration': "8.1s",
+ // }
+ // ]
+ //
+ updateProtobuf: "ChoIAiACKgwIARIICAQSBM9UdYs6BnN0YQB0ZRIECAwQCg==",
+ // The base64 of binary protobuf representation of response:
+ // {
+ // "matches": [
+ // {
+ // "threat_type": 2, // SOCIAL_ENGINEERING_PUBLIC
+ // "threat": {
+ // "hash": string,
+ // },
+ // "cacheDuration": "8.1",
+ // }
+ // ],
+ // "minimumWaitDuration": 12.0..1,
+ // "negativeCacheDuration": 12.0..1,
+ // }
+ fullhashProtobuf: "CiwIAhoiCiDPVHWLptJSc/UYiabk1/wo5OkJqbggiylVKISK28bfeSoECAwQChIECAwQChoECAwQCg==",
+ },
+];
+
+function addDataV4ToServer(list, type, data) {
+ return new Promise(function(resolve, reject) {
+ var xhr = new XMLHttpRequest;
+ let params = new URLSearchParams();
+ params.append("action", "store");
+ params.append("list", list);
+ params.append("type", type);
+ params.append("data", data);
+
+ xhr.open("PUT", "http://" + SJS + "?" + params.toString(), true);
+ xhr.setRequestHeader("Content-Type", "text/plain");
+ xhr.onreadystatechange = function() {
+ if (this.readyState == this.DONE) {
+ resolve();
+ }
+ };
+ xhr.send();
+ });
+}
+/**
+ * Grabs the results via XHR
+ */
+function checkResults(aTestdata, aExpected) {
+ let xhr = new XMLHttpRequest();
+ xhr.responseType = "text";
+ xhr.onload = function() {
+ is(aExpected, xhr.response, "Correct report request");
+ SimpleTest.finish();
+ };
+ xhr.onerror = function() {
+ ok(false, "Can't get results from server.");
+ SimpleTest.finish();
+ };
+ let params = new URLSearchParams();
+ params.append("action", "getreport");
+ params.append("list", aTestdata.list);
+ let url = "http://" + SJS + "?" + params.toString();
+
+ xhr.open("GET", url, true);
+ xhr.setRequestHeader("Content-Type", "text/plain");
+ xhr.send();
+}
+
+function waitForUpdate(data) {
+ listmanager.checkForUpdates(data.updateUrl);
+ return new Promise(resolve => {
+ Services.obs.addObserver(function observer(aSubject, aTopic) {
+ Services.obs.removeObserver(observer, aTopic);
+ resolve();
+ }, "safebrowsing-update-finished");
+ });
+}
+
+function addUpdateDataV4ToServer(list, data) {
+ return addDataV4ToServer(list, "update", data);
+}
+
+function addFullhashV4DataToServer(list, data) {
+ return addDataV4ToServer(list, "fullhash", data);
+}
+
+function setupTestData(data) {
+ let updateParams = new URLSearchParams();
+ updateParams.append("action", "get");
+ updateParams.append("list", data.list);
+ updateParams.append("type", "update");
+ data.updateUrl = "http://" + SJS + "?" + updateParams.toString();
+
+ let gethashParams = new URLSearchParams();
+ gethashParams.append("action", "get");
+ gethashParams.append("list", data.list);
+ gethashParams.append("type", "fullhash");
+ data.gethashUrl = "http://" + SJS + "?" + gethashParams.toString();
+
+ listmanager.registerTable(data.list,
+ data.provider,
+ data.updateUrl,
+ data.gethashUrl);
+
+ let promises = [];
+ let activeTablePref = "urlclassifier.phishTable";
+ let activeTable = SpecialPowers.getCharPref(activeTablePref);
+ activeTable += "," + data.list;
+
+ let reportPref = "browser.safebrowsing.provider." + data.provider + ".dataSharingURL";
+ let reportParams = new URLSearchParams();
+ reportParams.append("action", "report");
+ reportParams.append("list", data.list);
+ data.reportUrl = "http://" + SJS + "?" + reportParams.toString();
+
+ let reportEnabledPref = "browser.safebrowsing.provider." + data.provider + ".dataSharing.enabled";
+
+ promises.push(pushPrefs([reportPref, data.reportUrl]));
+ promises.push(pushPrefs([reportEnabledPref, true]));
+ promises.push(pushPrefs([activeTablePref, activeTable]));
+ promises.push(addUpdateDataV4ToServer(data.list, data.updateProtobuf));
+ promises.push(addFullhashV4DataToServer(data.list, data.fullhashProtobuf));
+ return Promise.all(promises);
+}
+
+function testOnWindow(aTestData) {
+ return new Promise(resolve => {
+ let win = mainWindow.OpenBrowserWindow();
+
+ (async function() {
+ await new Promise(rs => whenDelayedStartupFinished(win, rs));
+
+ let expected;
+ let browser = win.gBrowser.selectedBrowser;
+ let wp = win.gBrowser.contentDocument.docShell.QueryInterface(Ci.nsIWebProgress);
+ let progressListener = {
+ onSecurityChange(aWebProgress, aRequest, aState) {
+ let utils = Cc["@mozilla.org/url-classifier/utils;1"].
+ getService(Ci.nsIUrlClassifierUtils);
+ expected = aTestData.reportUrl + "&$req=" +
+ utils.makeThreatHitReport(aRequest,
+ aTestData.list,
+ btoa(hash(aTestData.url)));
+
+ },
+ QueryInterface: XPCOMUtils.generateQI(["nsISupportsWeakReference"])
+ };
+ wp.addProgressListener(progressListener, wp.NOTIFY_SECURITY);
+
+ await BrowserTestUtils.loadURI(browser, aTestData.url);
+ await BrowserTestUtils.waitForContentEvent(browser, "DOMContentLoaded");
+ checkResults(aTestData, expected);
+ win.close();
+ resolve();
+ })();
+ });
+}
+SpecialPowers.pushPrefEnv(
+ {"set": [["browser.safebrowsing.phishing.enabled", true]]},
+ test);
+
+function test() {
+ (async function() {
+ await classifierHelper.waitForInit();
+
+ for (let testData of testDatas) {
+ await setupTestData(testData);
+ await waitForUpdate(testData);
+ await testOnWindow(testData);
+ await classifierHelper._cleanup();
+ }
+ })();
+}
+
+SimpleTest.waitForExplicitFinish();
+
+</script>
+
+</pre>
+<iframe id="testFrame" width="100%" height="100%" onload=""></iframe>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/threathit.sjs
@@ -0,0 +1,87 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const CC = Components.Constructor;
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+ "nsIBinaryInputStream",
+ "setInputStream");
+
+function handleRequest(request, response)
+{
+ Components.utils.importGlobalProperties(["URLSearchParams"]);
+ let params = new URLSearchParams(request.queryString);
+ var action = params.get("action");
+
+ var responseBody;
+
+ // Store data in the server side.
+ if (action == "store") {
+ // In the server side we will store:
+ // All the full hashes or update for a given list
+ let state = params.get("list") + params.get("type");
+ let dataStr = params.get("data");
+ setState(state, dataStr);
+ return;
+ } else if (action == "get") {
+ let state = params.get("list") + params.get("type");
+ responseBody = base64ToString(getState(state));
+ response.setStatusLine(request.httpVersion, 200, "OK");
+ response.bodyOutputStream.write(responseBody,
+ responseBody.length);
+ } else if (action == "report") {
+ let state = params.get("list") + "report";
+ let requestUrl = request.scheme + "://" + request.host + ":" +
+ request.port + request.path + "?" + request.queryString;
+ setState(state, requestUrl);
+ } else if (action == "getreport") {
+ let state = params.get("list") + "report";
+ responseBody = getState(state);
+ response.setHeader("Content-Type", "text/plain", false);
+ response.write(responseBody);
+ }
+}
+
+var base64Pad = '=';
+/* Convert Base64 data to a string */
+var toBinaryTable = [
+ -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+ -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+ -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
+ 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1,
+ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
+ 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
+ -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
+ 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1
+];
+
+function base64ToString(data) {
+ var result = '';
+ var leftbits = 0; // number of bits decoded, but yet to be appended
+ var leftdata = 0; // bits decoded, but yet to be appended
+
+ // Convert one by one.
+ for (var i = 0; i < data.length; i++) {
+ var c = toBinaryTable[data.charCodeAt(i) & 0x7f];
+ var padding = (data[i] == base64Pad);
+ // Skip illegal characters and whitespace
+ if (c == -1) continue;
+ // Collect data into leftdata, update bitcount
+ leftdata = (leftdata << 6) | c;
+ leftbits += 6;
+
+ // If we have 8 or more bits, append 8 bits to the result
+ if (leftbits >= 8) {
+ leftbits -= 8;
+ // Append if not padding.
+ if (!padding)
+ result += String.fromCharCode((leftdata >> leftbits) & 0xff);
+ leftdata &= (1 << leftbits) - 1;
+ }
+ }
+
+ // If there are any bits left, the base64 string was corrupted
+ if (leftbits)
+ throw Components.Exception('Corrupted base64 string');
+
+ return result;
+}