Bug 1421803 - Send ThreatHit requests and telemetry at the right time. r?dimi draft
authorFrancois Marier <francois@mozilla.com>
Tue, 28 Nov 2017 12:10:37 -0800
changeset 706067 2ab708d9285d4aa1f48212947d4f2e956de5a60c
parent 705309 91fc3a79606bdc7fb43fef12eff7e65b5b84c00e
child 742554 cdacf518786f99fcfef6622dfd03c8c5f840bd87
push id91684
push userfmarier@mozilla.com
push dateFri, 01 Dec 2017 05:59:02 +0000
reviewersdimi
bugs1421803
milestone59.0a1
Bug 1421803 - Send ThreatHit requests and telemetry at the right time. r?dimi The ThreatHit requests were never being sent because SetMatchedInfo() was called on the channel _after_ calling SendThreatHitReport(). Additionally, the telemetry was sent in OnStartRequest() and so errors returned in OnStopRequest() would not be caught. This patch also includes some improvements to the logging of these requests which can be toggled using: MOZ_LOG="UrlClassifierDbService:5,nsChannelClassifier:5" MozReview-Commit-ID: 9dtRgEPVS3g
netwerk/base/nsChannelClassifier.cpp
netwerk/base/nsChannelClassifier.h
netwerk/base/nsIURIClassifier.idl
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
--- a/netwerk/base/nsChannelClassifier.cpp
+++ b/netwerk/base/nsChannelClassifier.cpp
@@ -1094,35 +1094,42 @@ nsChannelClassifier::IsTrackerWhiteliste
     LOG(("nsChannelClassifier[%p]:IsTrackerWhitelisted whitelist disabled",
          this));
     return NS_ERROR_TRACKING_URI;
   }
 
   return uriClassifier->AsyncClassifyLocalWithTables(aWhiteListURI, trackingWhitelist, aCallback);
 }
 
+/* static */
 nsresult
 nsChannelClassifier::SendThreatHitReport(nsIChannel *aChannel,
-                                         const nsACString& aProvider)
+                                         const nsACString& aProvider,
+                                         const nsACString& aList,
+                                         const nsACString& aFullHash)
 {
+  NS_ENSURE_ARG_POINTER(aChannel);
 
   nsAutoCString provider(aProvider);
   nsPrintfCString reportEnablePref("browser.safebrowsing.provider.%s.dataSharing.enabled",
                                    provider.get());
   if (!Preferences::GetBool(reportEnablePref.get(), false)) {
+    LOG(("nsChannelClassifier::SendThreatHitReport data sharing disabled for %s",
+         provider.get()));
     return NS_OK;
   }
 
   nsCOMPtr<nsIURIClassifier> uriClassifier =
     do_GetService(NS_URLCLASSIFIERDBSERVICE_CONTRACTID);
   if (!uriClassifier) {
     return NS_ERROR_UNEXPECTED;
   }
 
-  nsresult rv = uriClassifier->SendThreatHitReport(mChannel);
+  nsresult rv = uriClassifier->SendThreatHitReport(aChannel, aProvider, aList,
+                                                   aFullHash);
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsChannelClassifier::OnClassifyComplete(nsresult aErrorCode,
                                         const nsACString& aList,
@@ -1155,17 +1162,17 @@ nsChannelClassifier::OnClassifyComplete(
       // Do update the security state of the document and fire a security
       // change event.
       SetBlockedContent(mChannel, aErrorCode, aList, aProvider, aFullHash);
 
       if (aErrorCode == NS_ERROR_MALWARE_URI ||
           aErrorCode == NS_ERROR_PHISHING_URI ||
           aErrorCode == NS_ERROR_UNWANTED_URI ||
           aErrorCode == NS_ERROR_HARMFUL_URI) {
-        SendThreatHitReport(mChannel, aProvider);
+        SendThreatHitReport(mChannel, aProvider, aList, aFullHash);
       }
 
       mChannel->Cancel(aErrorCode);
     }
     LOG(("nsChannelClassifier[%p]: resuming channel %p from "
          "OnClassifyComplete", this, mChannel.get()));
     mChannel->Resume();
   }
--- a/netwerk/base/nsChannelClassifier.h
+++ b/netwerk/base/nsChannelClassifier.h
@@ -86,18 +86,20 @@ private:
     // by ShouldEnableTrackingProtection().
     nsresult ShouldEnableTrackingProtectionInternal(nsIChannel *aChannel,
                                                     bool aAnnotationsOnly,
                                                     bool *result);
 
     bool AddonMayLoad(nsIChannel *aChannel, nsIURI *aUri);
     void AddShutdownObserver();
     void RemoveShutdownObserver();
-    nsresult SendThreatHitReport(nsIChannel *aChannel,
-                                 const nsACString& aProvider);
+    static nsresult SendThreatHitReport(nsIChannel *aChannel,
+                                        const nsACString& aProvider,
+                                        const nsACString& aList,
+                                        const nsACString& aFullHash);
 public:
     // If we are blocking content, update the corresponding flag in the respective
     // docshell and call nsISecurityEventSink::onSecurityChange.
     static nsresult SetBlockedContent(nsIChannel *channel,
                                       nsresult aErrorCode,
                                       const nsACString& aList,
                                       const nsACString& aProvider,
                                       const nsACString& aFullHash);
--- a/netwerk/base/nsIURIClassifier.idl
+++ b/netwerk/base/nsIURIClassifier.idl
@@ -102,12 +102,19 @@ interface nsIURIClassifier : nsISupports
    */
   ACString classifyLocal(in nsIURI aURI, in ACString aTables);
 
   /**
    * Report to the provider that a Safe Browsing warning was shown.
    *
    * @param aChannel
    *        Channel for which the URL matched something on the threat list.
+   * @param aProvider
+   *        Provider to notify.
+   * @param aList
+   *        List where the full hash was found.
+   * @param aFullHash
+   *        Full URL hash that triggered the warning.
    */
 
-  void sendThreatHitReport(in nsIChannel aChannel);
+  void sendThreatHitReport(in nsIChannel aChannel, in ACString aProvider,
+                           in ACString aList, in ACString aFullHash);
 };
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -1,9 +1,9 @@
-//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
 #include "nsAppDirectoryServiceDefs.h"
 #include "nsArrayUtils.h"
@@ -1961,143 +1961,164 @@ public:
 
 private:
   ~ThreatHitReportListener() = default;
 };
 
 NS_IMPL_ISUPPORTS(ThreatHitReportListener, nsIStreamListener, nsIRequestObserver)
 
 NS_IMETHODIMP
-ThreatHitReportListener::OnStartRequest(nsIRequest* aRequest, nsISupports* aContext)
+ThreatHitReportListener::OnStartRequest(nsIRequest* aRequest,
+                                        nsISupports* aContext)
 {
+  if (!LOG_ENABLED()) {
+    return NS_OK; // Nothing to do!
+  }
+
   nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
-  if (httpChannel) {
-    nsresult rv;
-    nsresult status = NS_OK;
-    rv = httpChannel->GetStatus(&status);
-    NS_ENSURE_SUCCESS(rv, rv);
-
-    uint8_t netErrCode = NS_FAILED(status) ?
-      mozilla::safebrowsing::NetworkErrorToBucket(status) : 0;
-    mozilla::Telemetry::Accumulate(
-      mozilla::Telemetry::URLCLASSIFIER_THREATHIT_NETWORK_ERROR, netErrCode);
-
-    uint32_t requestStatus;
-    rv = httpChannel->GetResponseStatus(&requestStatus);
-    NS_ENSURE_SUCCESS(rv, rv);
-    mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_THREATHIT_REMOTE_STATUS,
-                                   mozilla::safebrowsing::HTTPStatusToBucket(requestStatus));
-    if (LOG_ENABLED()) {
-      nsAutoCString errorName, spec;
-      mozilla::GetErrorName(status, errorName);
-      nsCOMPtr<nsIURI> uri;
-      rv = httpChannel->GetURI(getter_AddRefs(uri));
-      if (NS_SUCCEEDED(rv) && uri) {
-        uri->GetAsciiSpec(spec);
-      }
-
-      nsCOMPtr<nsIURLFormatter> urlFormatter =
-      do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
-
-      // Trim sensitive log data
-      nsString trimmedSpec;
-      rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmedSpec);
-      if (NS_SUCCEEDED(rv)) {
-        LOG(("ThreatHitReportListener::OnStartRequest "
-             "(status=%s, uri=%s, this=%p)", errorName.get(),
-             NS_ConvertUTF16toUTF8(trimmedSpec).get(), this));
-
-      }
-    }
-
-    LOG(("ThreatHit report response %d %d", requestStatus, netErrCode));
+  NS_ENSURE_TRUE(httpChannel, NS_OK);
+
+  nsresult rv, status;
+  rv = httpChannel->GetStatus(&status);
+  NS_ENSURE_SUCCESS(rv, NS_OK);
+  nsAutoCString errorName;
+  mozilla::GetErrorName(status, errorName);
+
+  uint32_t requestStatus;
+  rv = httpChannel->GetResponseStatus(&requestStatus);
+  NS_ENSURE_SUCCESS(rv, NS_OK);
+
+  nsAutoCString spec;
+  nsCOMPtr<nsIURI> uri;
+  rv = httpChannel->GetURI(getter_AddRefs(uri));
+  if (NS_SUCCEEDED(rv) && uri) {
+    uri->GetAsciiSpec(spec);
   }
+  nsCOMPtr<nsIURLFormatter> urlFormatter =
+    do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
+  nsAutoString trimmed;
+  rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmed);
+  NS_ENSURE_SUCCESS(rv, NS_OK);
+
+  LOG(("ThreatHitReportListener::OnStartRequest "
+       "(status=%s, code=%d, uri=%s, this=%p)", errorName.get(),
+       requestStatus, NS_ConvertUTF16toUTF8(trimmed).get(), this));
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
 ThreatHitReportListener::OnDataAvailable(nsIRequest* aRequest,
-                                           nsISupports* aContext,
-                                           nsIInputStream* aInputStream,
-                                           uint64_t aOffset,
-                                           uint32_t aCount)
+                                         nsISupports* aContext,
+                                         nsIInputStream* aInputStream,
+                                         uint64_t aOffset,
+                                         uint32_t aCount)
 {
   return NS_OK;
 }
+
 NS_IMETHODIMP
-ThreatHitReportListener::OnStopRequest(nsIRequest* aRequest, nsISupports* aContext,
-                                         nsresult aStatus)
+ThreatHitReportListener::OnStopRequest(nsIRequest* aRequest,
+                                       nsISupports* aContext,
+                                       nsresult aStatus)
 {
+  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
+  NS_ENSURE_TRUE(httpChannel, aStatus);
+
+  uint8_t netErrCode = NS_FAILED(aStatus) ?
+    mozilla::safebrowsing::NetworkErrorToBucket(aStatus) : 0;
+  mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_THREATHIT_NETWORK_ERROR, netErrCode);
+
+  uint32_t requestStatus;
+  nsresult rv = httpChannel->GetResponseStatus(&requestStatus);
+  NS_ENSURE_SUCCESS(rv, aStatus);
+  mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_THREATHIT_REMOTE_STATUS,
+                                 mozilla::safebrowsing::HTTPStatusToBucket(requestStatus));
+
+  if (LOG_ENABLED()) {
+    nsAutoCString errorName;
+    mozilla::GetErrorName(aStatus, errorName);
+
+    nsAutoCString spec;
+    nsCOMPtr<nsIURI> uri;
+    rv = httpChannel->GetURI(getter_AddRefs(uri));
+    if (NS_SUCCEEDED(rv) && uri) {
+      uri->GetAsciiSpec(spec);
+    }
+    nsCOMPtr<nsIURLFormatter> urlFormatter =
+      do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
+    nsString trimmed;
+    rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmed);
+    NS_ENSURE_SUCCESS(rv, aStatus);
+
+    LOG(("ThreatHitReportListener::OnStopRequest "
+         "(status=%s, code=%d, uri=%s, this=%p)", errorName.get(),
+         requestStatus, NS_ConvertUTF16toUTF8(trimmed).get(), this));
+  }
+
   return aStatus;
 }
 
 NS_IMETHODIMP
-nsUrlClassifierDBService::SendThreatHitReport(nsIChannel *aChannel)
+nsUrlClassifierDBService::SendThreatHitReport(nsIChannel *aChannel,
+                                              const nsACString& aProvider,
+                                              const nsACString& aList,
+                                              const nsACString& aFullHash)
 {
   NS_ENSURE_ARG_POINTER(aChannel);
-  nsresult rv;
-
-  nsCOMPtr<nsIClassifiedChannel> classifiedChannel =
-    do_QueryInterface(aChannel, &rv);
-  NS_ENSURE_SUCCESS(rv, rv);
-  if (!classifiedChannel) {
+
+  if (aProvider.IsEmpty()) {
+    LOG(("nsUrlClassifierDBService::SendThreatHitReport missing provider"));
     return NS_ERROR_FAILURE;
   }
-
-  nsAutoCString provider;
-  rv = classifiedChannel->GetMatchedProvider(provider);
-  if (NS_FAILED(rv) || provider.IsEmpty()) {
+  if (aList.IsEmpty()) {
+    LOG(("nsUrlClassifierDBService::SendThreatHitReport missing list"));
     return NS_ERROR_FAILURE;
   }
-
-  nsAutoCString listName;
-  rv = classifiedChannel->GetMatchedList(listName);
-  if (NS_FAILED(rv) || listName.IsEmpty()) {
-    return NS_ERROR_FAILURE;
-  }
-
-  nsAutoCString fullHash;
-  rv = classifiedChannel->GetMatchedFullHash(fullHash);
-  if (NS_FAILED(rv) || fullHash.IsEmpty()) {
+  if (aFullHash.IsEmpty()) {
+    LOG(("nsUrlClassifierDBService::SendThreatHitReport missing fullhash"));
     return NS_ERROR_FAILURE;
   }
 
   nsPrintfCString reportUrlPref("browser.safebrowsing.provider.%s.dataSharingURL",
-                                provider.get());
+                                PromiseFlatCString(aProvider).get());
 
   nsCOMPtr<nsIURLFormatter> formatter(
     do_GetService("@mozilla.org/toolkit/URLFormatterService;1"));
   if (!formatter) {
     return NS_ERROR_UNEXPECTED;
   }
 
   nsString urlStr;
-  rv = formatter->FormatURLPref(NS_ConvertUTF8toUTF16(reportUrlPref), urlStr);
+  nsresult rv = formatter->FormatURLPref(NS_ConvertUTF8toUTF16(reportUrlPref), urlStr);
   NS_ENSURE_SUCCESS(rv, rv);
 
   if (urlStr.IsEmpty() || NS_LITERAL_STRING("about:blank").Equals(urlStr)) {
-    LOG(("%s is missing a ThreatHit data reporting URL.", provider.get()));
+    LOG(("%s is missing a ThreatHit data reporting URL.", PromiseFlatCString(aProvider).get()));
     return NS_OK;
   }
 
   nsCOMPtr<nsIUrlClassifierUtils> utilsService =
     do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
   if (!utilsService) {
     return NS_ERROR_FAILURE;
   }
 
   nsAutoCString reportBody;
-  rv = utilsService->MakeThreatHitReport(aChannel, listName, fullHash, reportBody);
+  rv = utilsService->MakeThreatHitReport(aChannel, aList, aFullHash, reportBody);
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsAutoCString reportUriStr = NS_ConvertUTF16toUTF8(urlStr);
   reportUriStr.Append("&$req=");
   reportUriStr.Append(reportBody);
 
+  LOG(("Sending the following ThreatHit report to %s about %s: %s",
+       PromiseFlatCString(aProvider).get(), PromiseFlatCString(aList).get(),
+       reportBody.get()));
+
   nsCOMPtr<nsIURI> reportURI;
   rv = NS_NewURI(getter_AddRefs(reportURI), reportUriStr);
   NS_ENSURE_SUCCESS(rv, rv);
 
   uint32_t loadFlags = nsIChannel::INHIBIT_CACHING |
                        nsIChannel::LOAD_BYPASS_CACHE;
 
   nsCOMPtr<nsIChannel> reportChannel;
--- a/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
+++ b/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
@@ -574,17 +574,16 @@ HashCompleterRequest.prototype = {
   handleResponseV4: function HCR_handleResponseV4() {
     let callback = {
       // onCompleteHashFound will be called for each fullhash found in
       // FullHashResponse.
       onCompleteHashFound: (aCompleteHash,
                             aTableNames,
                             aPerHashCacheDuration) => {
         log("V4 fullhash response complete hash found callback: " +
-            JSON.stringify(aCompleteHash) + ", " +
             aTableNames + ", CacheDuration(" + aPerHashCacheDuration + ")");
 
         // Filter table names which we didn't requested.
         let filteredTables = aTableNames.split(",").filter(name => {
           return this.tableNames.get(name);
         });
         if (0 === filteredTables.length) {
           log("ERROR: Got complete hash which is from unknown table.");