Bug 1329558 - Implement Minimum wait duration for V4 gethash draft
authorThomas Nguyen <tnguyen@mozilla.com>
Thu, 09 Feb 2017 16:56:44 +0800
changeset 481108 bf2c9f62ca7922f4d39dbd0f4aa35ccc29d9c1fa
parent 479651 af8a2573d0f1e9cc6f2ba0ab67d7a702a197f177
child 545115 6857292723b04d20eda92c9bd69d0dee6b99aa0b
push id44723
push usertnguyen@mozilla.com
push dateThu, 09 Feb 2017 08:58:32 +0000
bugs1329558
milestone54.0a1
Bug 1329558 - Implement Minimum wait duration for V4 gethash MozReview-Commit-ID: 7i9Wz7pq0yJ
toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp
toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
--- a/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
+++ b/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
@@ -8,16 +8,19 @@ const Cr = Components.results;
 const Cu = Components.utils;
 
 // COMPLETE_LENGTH and PARTIAL_LENGTH copied from nsUrlClassifierDBService.h,
 // they correspond to the length, in bytes, of a hash prefix and the total
 // hash.
 const COMPLETE_LENGTH = 32;
 const PARTIAL_LENGTH = 4;
 
+// Upper limit on the server response minimumWaitDuration
+const MIN_WAIT_DURATION_MAX_VALUE = 24 * 60 * 60 * 1000;
+
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/NetUtil.jsm");
 
 XPCOMUtils.defineLazyServiceGetter(this, 'gDbService',
                                    '@mozilla.org/url-classifier/dbservice;1',
                                    'nsIUrlClassifierDBService');
 
@@ -160,16 +163,19 @@ function HashCompleter() {
   this._pendingRequests = {};
 
   // A map of gethash URLs to RequestBackoff objects.
   this._backoffs = {};
 
   // Whether we have been informed of a shutdown by the shutdown event.
   this._shuttingDown = false;
 
+  // A map of gethash URLs to next gethash time in miliseconds
+  this._nextGethashTimeMs = {};
+
   Services.obs.addObserver(this, "quit-application", false);
 
 }
 
 HashCompleter.prototype = {
   classID: Components.ID("{9111de73-9322-4bfc-8b65-2b727f3e6ec8}"),
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIUrlClassifierHashCompleter,
                                          Ci.nsIRunnable,
@@ -205,29 +211,36 @@ HashCompleter.prototype = {
       var jslib = Cc["@mozilla.org/url-classifier/jslib;1"]
                   .getService().wrappedJSObject;
 
       // Using the V4 backoff algorithm for both V2 and V4. See bug 1273398.
       this._backoffs[aGethashUrl] = new jslib.RequestBackoffV4(
         10 /* keep track of max requests */,
         0  /* don't throttle on successful requests per time period */);
     }
+
+    if (!this._nextGethashTimeMs[aGethashUrl]) {
+      this._nextGethashTimeMs[aGethashUrl] = 0;
+    }
+
     // Start off this request. Without dispatching to a thread, every call to
     // complete makes an individual HTTP request.
     Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
   },
 
   // This is called after several calls to |complete|, or after the
   // currentRequest has finished.  It starts off the HTTP request by making a
   // |begin| call to the HashCompleterRequest.
   run: function() {
     // Clear everything on shutdown
     if (this._shuttingDown) {
       this._currentRequest = null;
       this._pendingRequests = null;
+      this._nextGethashTimeMs = null;
+
       for (var url in this._backoffs) {
         this._backoffs[url] = null;
       }
       throw Cr.NS_ERROR_NOT_INITIALIZED;
     }
 
     // If we don't have an in-flight request, make one
     let pendingUrls = Object.keys(this._pendingRequests);
@@ -251,17 +264,18 @@ HashCompleter.prototype = {
   // gethashUrl and fetch the next pending request, if there is one.
   finishRequest: function(url, aStatus) {
     this._backoffs[url].noteServerResponse(aStatus);
     Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
   },
 
   // Returns true if we can make a request from the given url, false otherwise.
   canMakeRequest: function(aGethashUrl) {
-    return this._backoffs[aGethashUrl].canMakeRequest();
+    return this._backoffs[aGethashUrl].canMakeRequest() &&
+           Date.now() >= this._nextGethashTimeMs[aGethashUrl];
   },
 
   // Notifies the RequestBackoff of a new request so we can throttle based on
   // max requests/time period. This must be called before a channel is opened,
   // and finishRequest must be called once the response is received.
   noteRequest: function(aGethashUrl) {
     return this._backoffs[aGethashUrl].noteRequest();
   },
@@ -551,16 +565,27 @@ HashCompleterRequest.prototype = {
       },
 
       onResponseParsed : (aMinWaitDuration,
                           aNegCacheDuration) => {
         log("V4 fullhash response parsed callback: " +
             "MinWaitDuration(" + aMinWaitDuration + "), " +
             "NegativeCacheDuration(" + aNegCacheDuration + ")");
 
+        let minWaitDuration = aMinWaitDuration;
+
+        if (aMinWaitDuration > MIN_WAIT_DURATION_MAX_VALUE) {
+          minWaitDuration = MIN_WAIT_DURATION_MAX_VALUE;
+        } else if (aMinWaitDuration < 0) {
+          minWaitDuration = 0;
+        }
+
+        this._completer._nextGethashTimeMs[this.gethashUrl] =
+          Date.now() + minWaitDuration;
+
         // TODO: Bug 1311935 - Implement v4 cache.
       },
     };
 
     gUrlUtil.parseFindFullHashResponseV4(this._response, callback);
   },
 
   // This parses a table entry in the response body and calls |handleItem|
@@ -646,16 +671,17 @@ HashCompleterRequest.prototype = {
               createInstance(Ci.nsIScriptableInputStream);
     sis.init(aInputStream);
     this._response += sis.readBytes(aCount);
   },
 
   onStartRequest: function HCR_onStartRequest(aRequest, aContext) {
     // At this point no data is available for us and we have no reason to
     // terminate the connection, so we do nothing until |onStopRequest|.
+    this._completer._nextGethashTimeMs[this.gethashUrl] = 0;
   },
 
   onStopRequest: function HCR_onStopRequest(aRequest, aContext, aStatusCode) {
     Services.obs.removeObserver(this, "quit-application");
 
     if (this.timer_) {
       this.timer_.cancel();
       this.timer_ = null;
--- a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp
@@ -425,17 +425,18 @@ nsUrlClassifierUtils::MakeFindFullHashRe
   aRequest = out;
 
   return NS_OK;
 }
 
 static uint32_t
 DurationToMs(const Duration& aDuration)
 {
-  return aDuration.seconds() * 1000 + aDuration.nanos() / 1000;
+  // Seconds precision is good enough. Ignore nanoseconds like Chrome does.
+  return aDuration.seconds() * 1000;
 }
 
 NS_IMETHODIMP
 nsUrlClassifierUtils::ParseFindFullHashResponseV4(const nsACString& aResponse,
                                                   nsIUrlClassifierParseFindFullHashCallback *aCallback)
 {
   enum CompletionErrorType {
     SUCCESS = 0,
--- a/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp
+++ b/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp
@@ -186,17 +186,17 @@ private:
     VerifyDuration(aPerHashCacheDuration, expected.mPerHashCacheDuration);
 
     mCallbackCount++;
   }
 
   void
   VerifyDuration(uint32_t aToVerify, const MyDuration& aExpected)
   {
-    ASSERT_TRUE(aToVerify == (aExpected.mSecs * 1000 + aExpected.mNanos / 1000));
+    ASSERT_TRUE(aToVerify == (aExpected.mSecs * 1000));
   }
 
   ~MyParseCallback() {}
 
   uint32_t& mCallbackCount;
 };
 
 NS_IMPL_ISUPPORTS(MyParseCallback, nsIUrlClassifierParseFindFullHashCallback)
--- a/toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
+++ b/toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
@@ -5,33 +5,61 @@ Cu.import("resource://gre/modules/Servic
 const TEST_TABLE_DATA_V4 = {
   tableName: "test-phish-proto",
   providerName: "google4",
   updateUrl: "http://localhost:5555/safebrowsing/update?",
   gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4?",
 };
 
 const PREF_NEXTUPDATETIME_V4 = "browser.safebrowsing.provider.google4.nextupdatetime";
+const GETHASH_PATH = "/safebrowsing/gethash-v4";
+
+// The protobuf binary represention of gethash response:
+// minimumWaitDuration : 12 secs 10 nanosecs
+// negativeCacheDuration : 120 secs 9 nanosecs
+//
+// { CompleteHash, ThreatType, CacheDuration { secs, nanos } };
+// { nsCString("01234567890123456789012345678901"), SOCIAL_ENGINEERING_PUBLIC, { 8, 500 } },
+// { nsCString("12345678901234567890123456789012"), SOCIAL_ENGINEERING_PUBLIC, { 7, 100} },
+// { nsCString("23456789012345678901234567890123"), SOCIAL_ENGINEERING_PUBLIC, { 1, 20 } },
+
+const GETHASH_RESPONSE_CONTENT = "\x0A\x2D\x08\x02\x1A\x22\x0A\x20\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x2A\x05\x08\x08\x10\xF4\x03\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x2A\x04\x08\x07\x10\x64\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x2A\x04\x08\x01\x10\x14\x12\x04\x08\x0C\x10\x0A\x1A\x04\x08\x78\x10\x09";
+
+// The protobuf binary represention of update response:
+//
+// [
+//   {
+//     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
+//     'response_type': 2, // FULL_UPDATE
+//     'new_client_state': 'sta\x00te', // NEW_CLIENT_STATE
+//     'checksum': { "sha256": CHECKSUM }, // CHECKSUM
+//     'additions': { 'compression_type': RAW,
+//                    'prefix_size': 4,
+//                    'raw_hashes': "00000001000000020000000300000004"}
+//   }
+// ]
+//
+const UPDATE_RESPONSE_CONTENT = "\x0A\x4A\x08\x02\x20\x02\x2A\x18\x08\x01\x12\x14\x08\x04\x12\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x3A\x06\x73\x74\x61\x00\x74\x65\x42\x22\x0A\x20\x30\x67\xC7\x2C\x5E\x50\x1C\x31\xE3\xFE\xCA\x73\xF0\x47\xDC\x34\x1A\x95\x63\x99\xEC\x70\x5E\x0A\xEE\x9E\xFB\x17\xA1\x55\x35\x78\x12\x08\x08\x08\x10\x80\x94\xEB\xDC\x03";
+const UPDATE_PATH = "/safebrowsing/update";
 
 let gListManager = Cc["@mozilla.org/url-classifier/listmanager;1"]
                      .getService(Ci.nsIUrlListManager);
 
 let gCompleter = Cc["@mozilla.org/url-classifier/hashcompleter;1"]
                     .getService(Ci.nsIUrlClassifierHashCompleter);
 
-XPCOMUtils.defineLazyServiceGetter(this, 'gUrlUtil',
-                                   '@mozilla.org/url-classifier/utils;1',
-                                   'nsIUrlClassifierUtils');
+XPCOMUtils.defineLazyServiceGetter(this, "gUrlUtil",
+                                   "@mozilla.org/url-classifier/utils;1",
+                                   "nsIUrlClassifierUtils");
 
 // Handles request for TEST_TABLE_DATA_V4.
 let gHttpServV4 = null;
-let gExpectedGetHashQueryV4 = "";
 
-const NEW_CLIENT_STATE = 'sta\0te';
-const CHECKSUM = '\x30\x67\xc7\x2c\x5e\x50\x1c\x31\xe3\xfe\xca\x73\xf0\x47\xdc\x34\x1a\x95\x63\x99\xec\x70\x5e\x0a\xee\x9e\xfb\x17\xa1\x55\x35\x78';
+const NEW_CLIENT_STATE = "sta\0te";
+const CHECKSUM = "\x30\x67\xc7\x2c\x5e\x50\x1c\x31\xe3\xfe\xca\x73\xf0\x47\xdc\x34\x1a\x95\x63\x99\xec\x70\x5e\x0a\xee\x9e\xfb\x17\xa1\x55\x35\x78";
 
 prefBranch.setBoolPref("browser.safebrowsing.debug", true);
 
 // The "\xFF\xFF" is to generate a base64 string with "/".
 prefBranch.setCharPref("browser.safebrowsing.id", "Firefox\xFF\xFF");
 
 // Register tables.
 gListManager.registerTable(TEST_TABLE_DATA_V4.tableName,
@@ -53,113 +81,137 @@ add_test(function test_update_v4() {
 });
 
 add_test(function test_getHashRequestV4() {
   let request = gUrlUtil.makeFindFullHashRequestV4([TEST_TABLE_DATA_V4.tableName],
                                                    [btoa(NEW_CLIENT_STATE)],
                                                    [btoa("0123"), btoa("1234567"), btoa("1111")],
                                                    1,
                                                    3);
-  gExpectedGetHashQueryV4 = '&$req=' + request;
-
+  registerHandlerGethashV4("&$req=" + request);
   let completeFinishedCnt = 0;
 
   gCompleter.complete("0123", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       equal(hash, "01234567890123456789012345678901");
       equal(table, TEST_TABLE_DATA_V4.tableName);
       equal(chunkId, 0);
       do_print("completion: " + hash + ", " + table + ", " + chunkId);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 
   gCompleter.complete("1234567", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       equal(hash, "12345678901234567890123456789012");
       equal(table, TEST_TABLE_DATA_V4.tableName);
       equal(chunkId, 0);
       do_print("completion: " + hash + ", " + table + ", " + chunkId);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 
   gCompleter.complete("1111", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       ok(false, "1111 is not the prefix of " + hash);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 });
 
-function run_test() {
-  gHttpServV4 = new HttpServer();
-  gHttpServV4.registerDirectory("/", do_get_cwd());
+add_test(function test_minWaitDuration() {
+  let failedComplete = function() {
+    gCompleter.complete("0123", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
+      completionFinished(status) {
+        equal(status, Cr.NS_ERROR_ABORT);
+      },
+    });
+  };
+
+  let successComplete = function() {
+    gCompleter.complete("1234567", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
+      completion(hash, table, chunkId) {
+        equal(hash, "12345678901234567890123456789012");
+        equal(table, TEST_TABLE_DATA_V4.tableName);
+        equal(chunkId, 0);
+        do_print("completion: " + hash + ", " + table + ", " + chunkId);
+      },
+
+      completionFinished(status) {
+        equal(status, Cr.NS_OK);
+        run_next_test();
+      },
+    });
+  };
 
+  let request = gUrlUtil.makeFindFullHashRequestV4([TEST_TABLE_DATA_V4.tableName],
+                                                   [btoa(NEW_CLIENT_STATE)],
+                                                   [btoa("1234567")],
+                                                   1,
+                                                   1);
+  registerHandlerGethashV4("&$req=" + request);
+
+  // The last gethash response contained a min wait duration 12 secs 10 nano
+  // So subsequent requests can happen only after the min wait duration
+  do_timeout(1000, failedComplete);
+  do_timeout(2000, failedComplete);
+  do_timeout(4000, failedComplete);
+  do_timeout(13000, successComplete);
+});
+
+function registerHandlerGethashV4(aExpectedQuery) {
+  gHttpServV4.registerPathHandler(GETHASH_PATH, null);
+  // V4 gethash handler.
+  gHttpServV4.registerPathHandler(GETHASH_PATH, function(request, response) {
+    equal(request.queryString, aExpectedQuery);
+
+    response.setStatusLine(request.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(GETHASH_RESPONSE_CONTENT,
+                                    GETHASH_RESPONSE_CONTENT.length);
+  });
+}
+
+function registerHandlerUpdateV4() {
   // Update handler. Will respond a valid state to be verified in the
   // gethash handler.
-  gHttpServV4.registerPathHandler("/safebrowsing/update", function(request, response) {
+  gHttpServV4.registerPathHandler(UPDATE_PATH, function(request, response) {
     response.setHeader("Content-Type",
                        "application/vnd.google.safebrowsing-update", false);
     response.setStatusLine(request.httpVersion, 200, "OK");
-
-    // The protobuf binary represention of response:
-    //
-    // [
-    //   {
-    //     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
-    //     'response_type': 2, // FULL_UPDATE
-    //     'new_client_state': 'sta\x00te', // NEW_CLIENT_STATE
-    //     'checksum': { "sha256": CHECKSUM }, // CHECKSUM
-    //     'additions': { 'compression_type': RAW,
-    //                    'prefix_size': 4,
-    //                    'raw_hashes': "00000001000000020000000300000004"}
-    //   }
-    // ]
-    //
-    let content = "\x0A\x4A\x08\x02\x20\x02\x2A\x18\x08\x01\x12\x14\x08\x04\x12\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x3A\x06\x73\x74\x61\x00\x74\x65\x42\x22\x0A\x20\x30\x67\xC7\x2C\x5E\x50\x1C\x31\xE3\xFE\xCA\x73\xF0\x47\xDC\x34\x1A\x95\x63\x99\xEC\x70\x5E\x0A\xEE\x9E\xFB\x17\xA1\x55\x35\x78\x12\x08\x08\x08\x10\x80\x94\xEB\xDC\x03";
-
-    response.bodyOutputStream.write(content, content.length);
+    response.bodyOutputStream.write(UPDATE_RESPONSE_CONTENT,
+                                    UPDATE_RESPONSE_CONTENT.length);
 
     waitUntilMetaDataSaved(NEW_CLIENT_STATE, CHECKSUM, () => {
       run_next_test();
     });
 
   });
-
-  // V4 gethash handler.
-  gHttpServV4.registerPathHandler("/safebrowsing/gethash-v4", function(request, response) {
-    equal(request.queryString, gExpectedGetHashQueryV4);
+}
 
-    // { nsCString("01234567890123456789012345678901"), SOCIAL_ENGINEERING_PUBLIC, { 8, 500 } },
-    // { nsCString("12345678901234567890123456789012"), SOCIAL_ENGINEERING_PUBLIC, { 7, 100} },
-    // { nsCString("23456789012345678901234567890123"), SOCIAL_ENGINEERING_PUBLIC, { 1, 20 } },
-    let content = "\x0A\x2D\x08\x02\x1A\x22\x0A\x20\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x2A\x05\x08\x08\x10\xF4\x03\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x2A\x04\x08\x07\x10\x64\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x2A\x04\x08\x01\x10\x14\x12\x04\x08\x0C\x10\x0A\x1A\x04\x08\x78\x10\x09";
+function run_test() {
+  gHttpServV4 = new HttpServer();
+  gHttpServV4.registerDirectory("/", do_get_cwd());
 
-    response.setStatusLine(request.httpVersion, 200, "OK");
-    response.bodyOutputStream.write(content, content.length);
-  });
-
+  registerHandlerUpdateV4();
   gHttpServV4.start(5555);
-
   run_next_test();
-}
\ No newline at end of file
+}