Bug 1305581 - P2. Testcase for verifying checksum when update. r?francois draft
authorDimiL <dlee@mozilla.com>
Fri, 07 Oct 2016 14:02:10 +0800
changeset 421986 dcdfeca4166256256ce124c1e7a61f6f4cc17b6b
parent 421985 2af3b0d24411cdc733415d7c8df54c1e4eaa1177
child 533217 34163f8c27b112ab284a05e9e2e53a0cfd47a40d
push id31648
push userdlee@mozilla.com
push dateFri, 07 Oct 2016 06:02:34 +0000
reviewersfrancois
bugs1305581
milestone52.0a1
Bug 1305581 - P2. Testcase for verifying checksum when update. r?francois MozReview-Commit-ID: LXnGgUnM038
toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
--- a/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
+++ b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
@@ -32,31 +32,60 @@ RemoveIntersection(const _PrefixArray& i
 static void
 RemoveElements(const nsTArray<uint32_t>& removal, _PrefixArray& outArray)
 {
   for (int32_t i = removal.Length() - 1; i >= 0; i--) {
     outArray.RemoveElementAt(removal[i]);
   }
 }
 
+static void
+MergeAndSortArray(const _PrefixArray& array1,
+                  const _PrefixArray& array2,
+                  _PrefixArray& output)
+{
+  output.Clear();
+  output.AppendElements(array1);
+  output.AppendElements(array2);
+  output.Sort();
+}
+
 // This function converts lexigraphic-sorted prefixes to a hashtable
 // which key is prefix size and value is concatenated prefix string.
 static void
 PrefixArrayToPrefixStringMap(const _PrefixArray& prefixArray,
                              PrefixStringMap& outMap)
 {
   outMap.Clear();
 
   for (uint32_t i = 0; i < prefixArray.Length(); i++) {
     const _Prefix& prefix = prefixArray[i];
     nsCString* prefixString = outMap.LookupOrAdd(prefix.Length());
     prefixString->Append(prefix.BeginReading(), prefix.Length());
   }
 }
 
+static void
+CalculateCheckSum(_PrefixArray& prefixArray, nsCString& checksum)
+{
+  prefixArray.Sort();
+
+  nsresult rv;
+  nsCOMPtr<nsICryptoHash> cryptoHash =
+    do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+
+  cryptoHash->Init(nsICryptoHash::SHA256);
+  for (uint32_t i = 0; i < prefixArray.Length(); i++) {
+    const _Prefix& prefix = prefixArray[i];
+    cryptoHash->Update(reinterpret_cast<uint8_t*>(
+                       const_cast<char*>(prefix.get())), prefix.Length());
+  }
+  cryptoHash->Finish(false, checksum);
+}
+
 // N: Number of prefixes, MIN/MAX: minimum/maximum prefix size
 // This function will append generated prefixes to outArray.
 static void
 CreateRandomSortedPrefixArray(uint32_t N,
                               uint32_t MIN,
                               uint32_t MAX,
                               _PrefixArray& outArray)
 {
@@ -98,30 +127,40 @@ CreateRandomRemovalIndices(uint32_t N,
     }
   }
 }
 
 // Function to generate TableUpdateV4.
 static void
 GenerateUpdateData(bool fullUpdate,
                    PrefixStringMap& add,
-                   nsTArray<uint32_t>& removal,
+                   nsTArray<uint32_t>* removal,
+                   nsCString* checksum,
                    nsTArray<TableUpdate*>& tableUpdates)
 {
   TableUpdateV4* tableUpdate = new TableUpdateV4(NS_LITERAL_CSTRING("gtest-malware-proto"));
   tableUpdate->SetFullUpdate(fullUpdate);
 
   for (auto iter = add.ConstIter(); !iter.Done(); iter.Next()) {
     nsCString* pstring = iter.Data();
     std::string str(pstring->BeginReading(), pstring->Length());
 
     tableUpdate->NewPrefixes(iter.Key(), str);
   }
 
-  tableUpdate->NewRemovalIndices(removal.Elements(), removal.Length());
+  if (removal) {
+    tableUpdate->NewRemovalIndices(removal->Elements(), removal->Length());
+  }
+
+  if (checksum) {
+    std::string stdChecksum;
+    stdChecksum.assign((char*)checksum->BeginReading(), checksum->Length());
+
+    tableUpdate->NewChecksum(stdChecksum);
+  }
 
   tableUpdates.AppendElement(tableUpdate);
 }
 
 static void
 VerifyPrefixSet(PrefixStringMap& expected)
 {
   // Verify the prefix set is written to disk.
@@ -190,360 +229,412 @@ testUpdate(nsTArray<TableUpdate*>& table
     nsresult rv = classifier->ApplyUpdates(&tableUpdates);
     ASSERT_TRUE(rv == NS_OK);
 
     VerifyPrefixSet(expected);
   });
 }
 
 static void
-testFullUpdate(PrefixStringMap& add)
+testFullUpdate(PrefixStringMap& add, nsCString* checksum)
 {
-  nsTArray<uint32_t> empty;
   nsTArray<TableUpdate*> tableUpdates;
-  GenerateUpdateData(true, add, empty, tableUpdates);
+
+  GenerateUpdateData(true, add, nullptr, checksum, tableUpdates);
 
   testUpdate(tableUpdates, add);
 }
 
 static void
 testPartialUpdate(PrefixStringMap& add,
-                  nsTArray<uint32_t>& removal,
+                  nsTArray<uint32_t>* removal,
+                  nsCString* checksum,
                   PrefixStringMap& expected)
 {
   nsTArray<TableUpdate*> tableUpdates;
-  GenerateUpdateData(false, add, removal, tableUpdates);
+  GenerateUpdateData(false, add, removal, checksum, tableUpdates);
 
   testUpdate(tableUpdates, expected);
 }
 
-
+// Testings start from here.
 TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate)
 {
   srand(time(NULL));
 
    _PrefixArray array;
   PrefixStringMap map;
+  nsCString checksum;
 
   CreateRandomSortedPrefixArray(5000, 4, 4, array);
   PrefixArrayToPrefixStringMap(array, map);
+  CalculateCheckSum(array, checksum);
 
-  testFullUpdate(map);
+  testFullUpdate(map, &checksum);
 
   Clear();
 }
 
-
 TEST(UrlClassifierTableUpdateV4, VariableLenghtPSetFullUpdate)
 {
    _PrefixArray array;
   PrefixStringMap map;
+  nsCString checksum;
 
   CreateRandomSortedPrefixArray(5000, 5, 32, array);
   PrefixArrayToPrefixStringMap(array, map);
+  CalculateCheckSum(array, checksum);
 
-  testFullUpdate(map);
+  testFullUpdate(map, &checksum);
 
   Clear();
 }
 
 // This test contain both variable length prefix set and fixed-length prefix set
 TEST(UrlClassifierTableUpdateV4, MixedPSetFullUpdate)
 {
    _PrefixArray array;
   PrefixStringMap map;
+  nsCString checksum;
 
   CreateRandomSortedPrefixArray(5000, 4, 4, array);
   CreateRandomSortedPrefixArray(1000, 5, 32, array);
   PrefixArrayToPrefixStringMap(array, map);
+  CalculateCheckSum(array, checksum);
 
-  testFullUpdate(map);
+  testFullUpdate(map, &checksum);
 
   Clear();
 }
 
 TEST(UrlClassifierTableUpdateV4, PartialUpdateWithRemoval)
 {
   _PrefixArray fArray, pArray, mergedArray;
   PrefixStringMap fMap, pMap, mergedMap;
+  nsCString checksum;
 
   {
     CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
     CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
     CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     // Remove 1/5 of elements of original prefix set.
     nsTArray<uint32_t> removal;
     CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
     RemoveElements(removal, fArray);
 
     // Calculate the expected prefix map.
-    mergedArray.AppendElements(fArray);
-    mergedArray.AppendElements(pArray);
-    mergedArray.Sort();
+    MergeAndSortArray(fArray, pArray, mergedArray);
     PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    CalculateCheckSum(mergedArray, checksum);
 
-    testPartialUpdate(pMap, removal, mergedMap);
+    testPartialUpdate(pMap, &removal, &checksum, mergedMap);
   }
 
   Clear();
 }
 
 TEST(UrlClassifierTableUpdateV4, PartialUpdateWithoutRemoval)
 {
   _PrefixArray fArray, pArray, mergedArray;
   PrefixStringMap fMap, pMap, mergedMap;
+  nsCString checksum;
 
   {
     CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
     CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
-    nsTArray<uint32_t> empty;
-
     CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     // Calculate the expected prefix map.
-    mergedArray.AppendElements(fArray);
-    mergedArray.AppendElements(pArray);
-    mergedArray.Sort();
+    MergeAndSortArray(fArray, pArray, mergedArray);
     PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    CalculateCheckSum(mergedArray, checksum);
 
-    testPartialUpdate(pMap, empty, mergedMap);
+    testPartialUpdate(pMap, nullptr, &checksum, mergedMap);
   }
 
   Clear();
 }
 
 // Expect failure because partial update contains prefix already
 // in old prefix set.
 TEST(UrlClassifierTableUpdateV4, PartialUpdatePrefixAlreadyExist)
 {
   _PrefixArray fArray, pArray;
   PrefixStringMap fMap, pMap;
+  nsCString checksum;
 
   {
     CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
-    nsTArray<uint32_t> empty;
     nsTArray<TableUpdate*> tableUpdates;
 
     // Pick one prefix from full update prefix and add it to partial update.
     // This should result a failure when call ApplyUpdates.
     pArray.AppendElement(fArray[rand() % fArray.Length()]);
     CreateRandomSortedPrefixArray(200, 4, 32, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
-    GenerateUpdateData(false, pMap, empty, tableUpdates);
+    GenerateUpdateData(false, pMap, nullptr, nullptr, tableUpdates);
     testUpdateFail(tableUpdates);
   }
 
   Clear();
 }
 
 // Test apply partial update directly without applying an full update first.
 TEST(UrlClassifierTableUpdateV4, OnlyPartialUpdate)
 {
   _PrefixArray pArray;
   PrefixStringMap pMap;
-  nsTArray<uint32_t> empty;
+  nsCString checksum;
 
   CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
   CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
   PrefixArrayToPrefixStringMap(pArray, pMap);
+  CalculateCheckSum(pArray, checksum);
 
-  testPartialUpdate(pMap, empty, pMap);
+  testPartialUpdate(pMap, nullptr, &checksum, pMap);
 
   Clear();
 }
 
 // Test partial update without any ADD prefixes, only removalIndices.
 TEST(UrlClassifierTableUpdateV4, PartialUpdateOnlyRemoval)
 {
   _PrefixArray fArray, pArray;
   PrefixStringMap fMap, pMap, mergedMap;
+  nsCString checksum;
 
   {
     CreateRandomSortedPrefixArray(5000, 4, 4, fArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
     // Remove 1/5 of elements of original prefix set.
     nsTArray<uint32_t> removal;
     CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
     RemoveElements(removal, fArray);
 
     PrefixArrayToPrefixStringMap(fArray, mergedMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testPartialUpdate(pMap, removal, mergedMap);
+    testPartialUpdate(pMap, &removal, &checksum, mergedMap);
   }
 
   Clear();
 }
 
 // Test one tableupdate array contains full update and multiple partial updates.
 TEST(UrlClassifierTableUpdateV4, MultipleTableUpdates)
 {
   _PrefixArray fArray, pArray, mergedArray;
   PrefixStringMap fMap, pMap, mergedMap;
+  nsCString checksum;
 
   {
-    nsTArray<uint32_t> empty;
     nsTArray<TableUpdate*> tableUpdates;
 
     // Generate first full udpate
     CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
     CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    GenerateUpdateData(true, fMap, empty, tableUpdates);
+    GenerateUpdateData(true, fMap, nullptr, &checksum, tableUpdates);
 
     // Generate second partial update
     CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
-    GenerateUpdateData(false, pMap, empty, tableUpdates);
+    MergeAndSortArray(fArray, pArray, mergedArray);
+    CalculateCheckSum(mergedArray, checksum);
+
+    GenerateUpdateData(false, pMap, nullptr, &checksum, tableUpdates);
 
     // Generate thrid partial update
     fArray.AppendElements(pArray);
     fArray.Sort();
     pArray.Clear();
     CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     // Remove 1/5 of elements of original prefix set.
     nsTArray<uint32_t> removal;
     CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
     RemoveElements(removal, fArray);
 
-    GenerateUpdateData(false, pMap, removal, tableUpdates);
+    MergeAndSortArray(fArray, pArray, mergedArray);
+    PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    CalculateCheckSum(mergedArray, checksum);
 
-    mergedArray.AppendElements(fArray);
-    mergedArray.AppendElements(pArray);
-    mergedArray.Sort();
-    PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates);
 
     testUpdate(tableUpdates, mergedMap);
   }
 
   Clear();
 }
 
 // Test apply full update first, and then apply multiple partial updates
 // in one tableupdate array.
 TEST(UrlClassifierTableUpdateV4, MultiplePartialUpdateTableUpdates)
 {
   _PrefixArray fArray, pArray, mergedArray;
   PrefixStringMap fMap, pMap, mergedMap;
+  nsCString checksum;
 
   {
     // Generate first full udpate
     CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
     CreateRandomSortedPrefixArray(3000, 5, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
     nsTArray<uint32_t> removal;
     nsTArray<TableUpdate*> tableUpdates;
 
     // Generate first partial update
     CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     // Remove 1/5 of elements of original prefix set.
     CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
     RemoveElements(removal, fArray);
 
-    GenerateUpdateData(false, pMap, removal, tableUpdates);
+    MergeAndSortArray(fArray, pArray, mergedArray);
+    CalculateCheckSum(mergedArray, checksum);
+
+    GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates);
 
     fArray.AppendElements(pArray);
     fArray.Sort();
     pArray.Clear();
     removal.Clear();
 
     // Generate second partial update.
     CreateRandomSortedPrefixArray(2000, 4, 4, pArray);
     CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     // Remove 1/5 of elements of original prefix set.
     CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
     RemoveElements(removal, fArray);
 
-    GenerateUpdateData(false, pMap, removal, tableUpdates);
+    MergeAndSortArray(fArray, pArray, mergedArray);
+    PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    CalculateCheckSum(mergedArray, checksum);
 
-    mergedArray.AppendElements(fArray);
-    mergedArray.AppendElements(pArray);
-    mergedArray.Sort();
-    PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+    GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates);
 
     testUpdate(tableUpdates, mergedMap);
   }
 
   Clear();
 }
 
 // Test removal indices are larger than the original prefix set.
 TEST(UrlClassifierTableUpdateV4, RemovalIndexTooLarge)
 {
   _PrefixArray fArray, pArray;
   PrefixStringMap fMap, pMap;
+  nsCString checksum;
 
   {
     CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
     PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
 
-    testFullUpdate(fMap);
+    testFullUpdate(fMap, &checksum);
   }
 
   {
     nsTArray<uint32_t> removal;
     nsTArray<TableUpdate*> tableUpdates;
 
     CreateRandomSortedPrefixArray(200, 4, 32, pArray);
     RemoveIntersection(fArray, pArray);
     PrefixArrayToPrefixStringMap(pArray, pMap);
 
     for (uint32_t i = 0; i < fArray.Length() + 1 ;i++) {
       removal.AppendElement(i);
     }
 
-    GenerateUpdateData(false, pMap, removal, tableUpdates);
+    GenerateUpdateData(false, pMap, &removal, nullptr, tableUpdates);
     testUpdateFail(tableUpdates);
   }
 
   Clear();
 }
+
+TEST(UrlClassifierTableUpdateV4, InconsistChecksum)
+{
+  _PrefixArray fArray, pArray;
+  PrefixStringMap fMap, pMap;
+  nsCString checksum;
+
+  {
+    CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+    PrefixArrayToPrefixStringMap(fArray, fMap);
+    CalculateCheckSum(fArray, checksum);
+
+    testFullUpdate(fMap, &checksum);
+  }
+
+  {
+    nsTArray<TableUpdate*> tableUpdates;
+
+    CreateRandomSortedPrefixArray(200, 4, 32, pArray);
+    PrefixArrayToPrefixStringMap(pArray, pMap);
+
+    // Checksum should be calculated with both old prefix set and add prefix set,
+    // here we only calculate checksum with add prefix set to check if applyUpdate
+    // will return failure.
+    CalculateCheckSum(pArray, checksum);
+
+    GenerateUpdateData(false, pMap, nullptr, &checksum, tableUpdates);
+    testUpdateFail(tableUpdates);
+  }
+
+  Clear();
+}