--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -5130,26 +5130,35 @@ pref("urlclassifier.malwareTable", "goog
// In the official build, we are allowed to use google's private
// phishing list "goog-phish-shavar". See Bug 1288840.
pref("urlclassifier.phishTable", "goog-phish-shavar,test-phish-simple");
#else
pref("urlclassifier.phishTable", "googpub-phish-shavar,test-phish-simple");
#endif
// Tables for application reputation.
+#ifdef NIGHTLY_BUILD
+pref("urlclassifier.downloadBlockTable", "goog-badbinurl-shavar,goog-badbinurl-proto");
+#else
pref("urlclassifier.downloadBlockTable", "goog-badbinurl-shavar");
+#endif
#ifdef XP_WIN
// Only download the whitelist on Windows, since the whitelist is
// only useful for suppressing remote lookups for signed binaries which we can
// only verify on Windows (Bug 974579). Other platforms always do remote lookups.
+#ifdef NIGHTLY_BUILD
+pref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256,goog-downloadwhite-proto");
+#else
pref("urlclassifier.downloadAllowTable", "goog-downloadwhite-digest256");
+#endif // NIGHTLY_BUILD
+
#else
pref("urlclassifier.downloadAllowTable", "");
-#endif
+#endif // XP_WIN
pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,test-block-simple,test-flashallow-simple,testexcept-flashallow-simple,test-flash-simple,testexcept-flash-simple,test-flashsubdoc-simple,testexcept-flashsubdoc-simple,goog-downloadwhite-digest256,base-track-digest256,mozstd-trackwhite-digest256,content-track-digest256,mozplugin-block-digest256,mozplugin2-block-digest256");
// The table and update/gethash URLs for Safebrowsing phishing and malware
// checks.
pref("urlclassifier.trackingTable", "test-track-simple,base-track-digest256");
pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozstd-trackwhite-digest256");
@@ -5191,17 +5200,17 @@ pref("browser.safebrowsing.provider.goog
pref("browser.safebrowsing.provider.google.gethashURL", "https://safebrowsing.google.com/safebrowsing/gethash?client=SAFEBROWSING_ID&appver=%MAJOR_VERSION%&pver=2.2");
pref("browser.safebrowsing.provider.google.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
pref("browser.safebrowsing.provider.google.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.provider.google.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
// Prefs for v4.
pref("browser.safebrowsing.provider.google4.pver", "4");
-pref("browser.safebrowsing.provider.google4.lists", "goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto");
+pref("browser.safebrowsing.provider.google4.lists", "goog-badbinurl-proto,goog-downloadwhite-proto,goog-phish-proto,googpub-phish-proto,goog-malware-proto,goog-unwanted-proto");
pref("browser.safebrowsing.provider.google4.updateURL", "https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch?$ct=application/x-protobuf&key=%GOOGLE_API_KEY%");
// Leave it empty until we roll out v4 hash completion feature. See Bug 1323856.
pref("browser.safebrowsing.provider.google4.gethashURL", "");
pref("browser.safebrowsing.provider.google4.reportURL", "https://safebrowsing.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
pref("browser.safebrowsing.provider.google4.reportPhishMistakeURL", "https://%LOCALE%.phish-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.provider.google4.reportMalwareMistakeURL", "https://%LOCALE%.malware-error.mozilla.com/?hl=%LOCALE%&url=");
pref("browser.safebrowsing.reportPhishURL", "https://%LOCALE%.phish-report.mozilla.com/?hl=%LOCALE%&url=");
--- a/toolkit/components/downloads/ApplicationReputation.cpp
+++ b/toolkit/components/downloads/ApplicationReputation.cpp
@@ -49,17 +49,19 @@
#include "nsTArray.h"
#include "nsThreadUtils.h"
#include "nsXPCOMStrings.h"
#include "nsIContentPolicy.h"
#include "nsILoadInfo.h"
#include "nsContentUtils.h"
#include "nsWeakReference.h"
+#include "nsCharSeparatedTokenizer.h"
+using namespace mozilla::downloads;
using mozilla::ArrayLength;
using mozilla::BasePrincipal;
using mozilla::OriginAttributes;
using mozilla::Preferences;
using mozilla::TimeStamp;
using mozilla::Telemetry::Accumulate;
using safe_browsing::ClientDownloadRequest;
using safe_browsing::ClientDownloadRequest_CertificateChain;
@@ -82,16 +84,108 @@ using safe_browsing::ClientDownloadReque
#define PREF_BLOCK_POTENTIALLY_UNWANTED "browser.safebrowsing.downloads.remote.block_potentially_unwanted"
#define PREF_BLOCK_UNCOMMON "browser.safebrowsing.downloads.remote.block_uncommon"
// MOZ_LOG=ApplicationReputation:5
mozilla::LazyLogModule ApplicationReputationService::prlog("ApplicationReputation");
#define LOG(args) MOZ_LOG(ApplicationReputationService::prlog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(ApplicationReputationService::prlog, mozilla::LogLevel::Debug)
+namespace mozilla {
+namespace downloads {
+
+enum class TelemetryMatchInfo : uint8_t
+{
+ eNoMatch = 0x00,
+ eV2Match = 0x01,
+ eV4Match = 0x02,
+ eBothMatch = eV2Match | eV4Match,
+};
+
+MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(TelemetryMatchInfo)
+
+// Given a comma-separated list of tables which matched a URL, check to see if
+// at least one of these tables is present in the given pref.
+bool
+LookupTablesInPrefs(const nsACString& tables, const char* aPref)
+{
+ nsAutoCString prefList;
+ Preferences::GetCString(aPref, &prefList);
+ if (prefList.IsEmpty()) {
+ return false;
+ }
+
+ // Check if V2 and V4 are enabled in preference
+ // If V2 and V4 are both enabled, then we should do a telemetry record
+ // Both V2 and V4 begin with "goog" but V4 ends with "-proto"
+ nsCCharSeparatedTokenizer prefTokens(prefList, ',');
+ nsCString prefToken;
+ bool isV4Enabled = false;
+ bool isV2Enabled = false;
+
+ while (prefTokens.hasMoreTokens()) {
+ prefToken = prefTokens.nextToken();
+ if (StringBeginsWith(prefToken, NS_LITERAL_CSTRING("goog"))) {
+ if (StringEndsWith(prefToken, NS_LITERAL_CSTRING("-proto"))) {
+ isV4Enabled = true;
+ } else {
+ isV2Enabled = true;
+ }
+ }
+ }
+
+ bool shouldRecordTelemetry = isV2Enabled && isV4Enabled;
+ TelemetryMatchInfo telemetryInfo = TelemetryMatchInfo::eNoMatch;
+
+ // Parsed tables separated by "," into tokens then lookup each token
+ // in preference list
+ nsCCharSeparatedTokenizer tokens(tables, ',');
+ nsCString table;
+ bool found = false;
+
+ while (tokens.hasMoreTokens()) {
+ table = tokens.nextToken();
+ if (table.IsEmpty()) {
+ continue;
+ }
+
+ if (!FindInReadable(table, prefList)) {
+ continue;
+ }
+ found = true;
+
+ if (!shouldRecordTelemetry) {
+ return found;
+ }
+
+ // We are checking if the table found is V2 or V4 to record telemetry
+ // Both V2 and V4 begin with "goog" but V4 ends with "-proto"
+ if (StringBeginsWith(prefToken, NS_LITERAL_CSTRING("goog"))) {
+ if (StringEndsWith(prefToken, NS_LITERAL_CSTRING("-proto"))) {
+ telemetryInfo |= TelemetryMatchInfo::eV4Match;
+ } else {
+ telemetryInfo |= TelemetryMatchInfo::eV2Match;
+ }
+ }
+ }
+
+ // Record telemetry for matching allow list and block list
+ if (!strcmp(aPref, PREF_DOWNLOAD_BLOCK_TABLE)) {
+ Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_BLOCKLIST_MATCH,
+ static_cast<uint8_t>(telemetryInfo));
+ } else if (!strcmp(aPref, PREF_DOWNLOAD_ALLOW_TABLE)) {
+ Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_ALLOWLIST_MATCH,
+ static_cast<uint8_t>(telemetryInfo));
+ }
+
+ return found;
+}
+} // namespace downloads
+} // namespace mozilla
+
class PendingDBLookup;
// A single use class private to ApplicationReputationService encapsulating an
// nsIApplicationReputationQuery and an nsIApplicationReputationCallback. Once
// created by ApplicationReputationService, it is guaranteed to call mCallback.
// This class is private to ApplicationReputationService.
class PendingLookup final : public nsIStreamListener,
public nsITimerCallback,
@@ -343,29 +437,25 @@ PendingDBLookup::LookupSpecInternal(cons
NS_IMETHODIMP
PendingDBLookup::HandleEvent(const nsACString& tables)
{
// HandleEvent is guaranteed to call either:
// 1) PendingLookup::OnComplete if the URL matches the blocklist, or
// 2) PendingLookup::LookupNext if the URL does not match the blocklist.
// Blocklisting trumps allowlisting.
- nsAutoCString blockList;
- Preferences::GetCString(PREF_DOWNLOAD_BLOCK_TABLE, &blockList);
- if (!mAllowlistOnly && FindInReadable(blockList, tables)) {
+ if (!mAllowlistOnly && LookupTablesInPrefs(tables, PREF_DOWNLOAD_BLOCK_TABLE)) {
mPendingLookup->mBlocklistCount++;
Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, BLOCK_LIST);
LOG(("Found principal %s on blocklist [this = %p]", mSpec.get(), this));
return mPendingLookup->OnComplete(true, NS_OK,
nsIApplicationReputationService::VERDICT_DANGEROUS);
}
- nsAutoCString allowList;
- Preferences::GetCString(PREF_DOWNLOAD_ALLOW_TABLE, &allowList);
- if (FindInReadable(allowList, tables)) {
+ if (LookupTablesInPrefs(tables, PREF_DOWNLOAD_ALLOW_TABLE)) {
mPendingLookup->mAllowlistCount++;
Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, ALLOW_LIST);
LOG(("Found principal %s on allowlist [this = %p]", mSpec.get(), this));
// Don't call onComplete, since blocklisting trumps allowlisting
} else {
LOG(("Didn't find principal %s on any list [this = %p]", mSpec.get(),
this));
Accumulate(mozilla::Telemetry::APPLICATION_REPUTATION_LOCAL, NO_LIST);
--- a/toolkit/components/downloads/ApplicationReputation.h
+++ b/toolkit/components/downloads/ApplicationReputation.h
@@ -15,16 +15,24 @@
#include "nsCOMPtr.h"
#include "nsString.h"
#include "mozilla/Logging.h"
class nsIRequest;
class PendingDBLookup;
class PendingLookup;
+namespace mozilla {
+namespace downloads {
+
+bool LookupTablesInPrefs(const nsACString& tables, const char* aPref);
+
+} // namespace downloads
+} // namespace mozilla
+
class ApplicationReputationService final :
public nsIApplicationReputationService {
public:
NS_DECL_ISUPPORTS
NS_DECL_NSIAPPLICATIONREPUTATIONSERVICE
public:
static ApplicationReputationService* GetSingleton();
--- a/toolkit/components/downloads/moz.build
+++ b/toolkit/components/downloads/moz.build
@@ -14,17 +14,17 @@ with Files('chromium/*'):
BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
with Files('generate_csd.sh'):
BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
with Files('nsIApplicationReputation.idl'):
BUG_COMPONENT = ('Toolkit', 'Safe Browsing')
-XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
+TEST_DIRS += ['test']
XPIDL_SOURCES += [
'nsIApplicationReputation.idl',
'nsIDownload.idl',
'nsIDownloadManager.idl',
'nsIDownloadManagerUI.idl',
'nsIDownloadProgressListener.idl',
]
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/gtest/TestLookupTable.cpp
@@ -0,0 +1,47 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "ApplicationReputation.h"
+#include "mozilla/Preferences.h"
+#include "nsLiteralString.h"
+
+using namespace mozilla;
+using namespace mozilla::downloads;
+
+TEST(PendingLookup, LookupTablesInPrefs)
+{
+ EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", "goog-badbinurl-proto,goog-downloadwhite-proto,goog-badbinurl-shavar"));
+
+ bool result;
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,adafaf,,daf,goog-badbinurl-proto"), "gtest.test");
+ ASSERT_TRUE(result);
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,adafaf,,daf,goog-downloadwhite-proto"), "gtest.test");
+ ASSERT_TRUE(result);
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto"), "gtest.test");
+ ASSERT_FALSE(result);
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+ ASSERT_TRUE(result);
+
+ EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", "goog-badbinurl-proto"));
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-badbinurl-proto"), "gtest.test");
+ ASSERT_TRUE(result);
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+ ASSERT_TRUE(result);
+
+ // Empty prefrence
+ EXPECT_EQ(NS_OK, Preferences::SetCString("gtest.test", ""));
+
+ result = LookupTablesInPrefs(NS_LITERAL_CSTRING("goog-phish-proto,goog-badbinurl-proto,goog-phish-shavar"), "gtest.test");
+ ASSERT_FALSE(result);
+
+ Preferences::ClearUser("gtest.test");
+}
+
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/gtest/moz.build
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+LOCAL_INCLUDES += [
+ '../..',
+]
+
+UNIFIED_SOURCES += [
+ 'TestLookupTable.cpp',
+]
+
+FINAL_LIBRARY = 'xul-gtest'
new file mode 100644
--- /dev/null
+++ b/toolkit/components/downloads/test/moz.build
@@ -0,0 +1,10 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
+
+if CONFIG['ENABLE_TESTS']:
+ DIRS += ['gtest']
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -116,16 +116,32 @@
},
"APPLICATION_REPUTATION_REMOTE_LOOKUP_TIMEOUT": {
"alert_emails": ["safebrowsing-telemetry@mozilla.org"],
"expires_in_version": "56",
"kind": "boolean",
"bug_numbers": [1172689],
"description": "Recorded when application reputation remote lookup is performed, `true` is recorded if the lookup times out."
},
+ "APPLICATION_REPUTATION_BLOCKLIST_MATCH": {
+ "alert_emails": ["safebrowsing-telemetry@mozilla.org"],
+ "expires_in_version": "60",
+ "kind": "enumerated",
+ "n_values": 4,
+ "bug_numbers": [1331139],
+ "description": "For each Application Reputation lookup against both the V2 and V4 Google lists, note which version of block list returned a match (0 = no match, 1 = match only V2, 2 = match only V4, 3 = match both V2 and V4)"
+ },
+ "APPLICATION_REPUTATION_ALLOWLIST_MATCH": {
+ "alert_emails": ["safebrowsing-telemetry@mozilla.org"],
+ "expires_in_version": "60",
+ "kind": "enumerated",
+ "n_values": 4,
+ "bug_numbers": [1331139],
+ "description": "For each Application Reputation lookup against both the V2 and V4 Google lists, note which version of the allow list returned a match (0 = no match, 1 = match only V2, 2 = match only V4, 3 = match both V2 and V4)"
+ },
"AUDIOSTREAM_FIRST_OPEN_MS": {
"expires_in_version": "50",
"kind": "exponential",
"high": 10000,
"n_buckets": 50,
"description": "The length of time (in milliseconds) for the first open of AudioStream."
},
"AUDIOSTREAM_LATER_OPEN_MS": {
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
@@ -226,16 +226,20 @@ static const struct {
const char* mListName;
uint32_t mThreatType;
} THREAT_TYPE_CONV_TABLE[] = {
{ "goog-malware-proto", MALWARE_THREAT}, // 1
{ "googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2
{ "goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3
{ "goog-phish-proto", SOCIAL_ENGINEERING}, // 5
+ // For application reputation
+ { "goog-badbinurl-proto", MALICIOUS_BINARY}, // 7
+ { "goog-downloadwhite-proto", CSD_DOWNLOAD_WHITELIST}, // 9
+
// For testing purpose.
{ "test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2
{ "test-unwanted-proto", UNWANTED_SOFTWARE}, // 3
};
NS_IMETHODIMP
nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType,
nsACString& aListNames)