--- a/toolkit/components/url-classifier/Classifier.cpp
+++ b/toolkit/components/url-classifier/Classifier.cpp
@@ -29,16 +29,17 @@
// MOZ_LOG=UrlClassifierDbService:5
extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
#define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
#define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
#define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
+#define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
namespace mozilla {
namespace safebrowsing {
namespace {
@@ -175,16 +176,23 @@ Classifier::SetupPathNames()
// Directory where to move a backup before an update.
rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory));
NS_ENSURE_SUCCESS(rv, rv);
rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
+ // Directory where to be working on the update.
+ rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
+ NS_ENSURE_SUCCESS(rv, rv);
+
// Directory where to move the backup so we can atomically
// delete (really move) it.
rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
NS_ENSURE_SUCCESS(rv, rv);
rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
@@ -220,19 +228,29 @@ Classifier::Open(nsIFile& aCacheDirector
nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory));
NS_ENSURE_SUCCESS(rv, rv);
// Create the handles to the update and backup directories.
rv = SetupPathNames();
NS_ENSURE_SUCCESS(rv, rv);
// Clean up any to-delete directories that haven't been deleted yet.
+ // This is still required for backward compatibility.
rv = CleanToDelete();
NS_ENSURE_SUCCESS(rv, rv);
+ // If we met a crash during the previous update, "safebrowsing-updating"
+ // directory will exist and let's remove it.
+ rv = mUpdatingDirectory->Remove(true);
+ if (NS_SUCCEEDED(rv)) {
+ // If the "safebrowsing-updating" exists, it implies a crash occurred
+ // in the previous update.
+ LOG(("We may have hit a crash in the previous update."));
+ }
+
// Check whether we have an incomplete update and recover from the
// backup if so.
rv = RecoverBackups();
NS_ENSURE_SUCCESS(rv, rv);
// Make sure the main store directory exists.
rv = CreateStoreDirectory();
NS_ENSURE_SUCCESS(rv, rv);
@@ -255,16 +273,17 @@ Classifier::Close()
void
Classifier::Reset()
{
DropStores();
mRootStoreDirectory->Remove(true);
mBackupDirectory->Remove(true);
+ mUpdatingDirectory->Remove(true);
mToDeleteDirectory->Remove(true);
CreateStoreDirectory();
mTableFreshness.Clear();
RegenActiveTables();
}
@@ -342,21 +361,16 @@ Classifier::AbortUpdateAndReset(const ns
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
return;
}
LOG(("Abort updating table %s.", aTable.get()));
// ResetTables will clear both in-memory & on-disk data.
ResetTables(Clear_All, nsTArray<nsCString> { aTable });
-
- // Remove the backup and delete directory since we are aborting
- // from an update.
- Unused << RemoveBackupTables();
- Unused << CleanToDelete();
}
void
Classifier::TableRequest(nsACString& aResult)
{
MOZ_ASSERT(!NS_IsMainThread(),
"TableRequest must be called on the classifier worker thread.");
@@ -522,19 +536,154 @@ Classifier::Check(const nsACString& aSpe
Telemetry::Accumulate(Telemetry::URLCLASSIFIER_PREFIX_MATCH,
static_cast<uint8_t>(matchingStatistics));
}
}
return NS_OK;
}
+static nsresult
+SwapDirectoryContent(nsIFile* aDir1,
+ nsIFile* aDir2,
+ nsIFile* aParentDir,
+ nsIFile* aTempDir)
+{
+ // Pre-condition: |aDir1| and |aDir2| are directory and their parent
+ // are both |aParentDir|.
+ //
+ // Post-condition: The locations where aDir1 and aDir2 point to will not
+ // change but their contents will be exchanged. If we failed
+ // to swap their content, everything will be rolled back.
+
+ nsAutoCString tempDirName;
+ aTempDir->GetNativeLeafName(tempDirName);
+
+ nsresult rv;
+
+ nsAutoCString dirName1, dirName2;
+ aDir1->GetNativeLeafName(dirName1);
+ aDir2->GetNativeLeafName(dirName2);
+
+ LOG(("Swapping directories %s and %s...", dirName1.get(),
+ dirName2.get()));
+
+ // 1. Rename "dirName1" to "temp"
+ rv = aDir1->RenameToNative(nullptr, tempDirName);
+ if (NS_FAILED(rv)) {
+ LOG(("Unable to rename %s to %s", dirName1.get(),
+ tempDirName.get()));
+ return rv; // Nothing to roll back.
+ }
+
+ // 1.1. Create a handle for temp directory. This is required since
+ // |nsIFile.rename| will not change the location where the
+ // object points to.
+ nsCOMPtr<nsIFile> tempDirectory;
+ rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
+ rv = tempDirectory->AppendNative(tempDirName);
+
+ // 2. Rename "dirName2" to "dirName1".
+ rv = aDir2->RenameToNative(nullptr, dirName1);
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
+ dirName2.get(), dirName1.get(), dirName1.get()));
+ nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
+ NS_ENSURE_SUCCESS(rbrv, rbrv);
+ return rv;
+ }
+
+ // 3. Rename "temp" to "dirName2".
+ rv = tempDirectory->RenameToNative(nullptr, dirName2);
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
+ // We've done (1) renaming "dir1 to temp" and
+ // (2) renaming "dir2 to dir1"
+ // so the rollback is
+ // (1) renaming "dir1 to dir2" and
+ // (2) renaming "temp to dir1"
+ nsresult rbrv; // rollback result
+ rbrv = aDir1->RenameToNative(nullptr, dirName2);
+ NS_ENSURE_SUCCESS(rbrv, rbrv);
+ rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
+ NS_ENSURE_SUCCESS(rbrv, rbrv);
+ return rv;
+ }
+
+ return rv;
+}
+
+void
+Classifier::RemoveUpdateIntermediaries()
+{
+ // Remove old LookupCaches.
+ for (auto c: mNewLookupCaches) {
+ delete c;
+ }
+ mNewLookupCaches.Clear();
+
+ // Remove the "old" directory. (despite its looking-new name)
+ if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
+ // If the directory is locked from removal for some reason,
+ // we will fail here and it doesn't matter until the next
+ // update. (the next udpate will fail due to the removable
+ // "safebrowsing-udpating" directory.)
+ LOG(("Failed to remove updating directory."));
+ }
+}
+
+nsresult
+Classifier::SwapInNewTablesAndCleanup()
+{
+ nsresult rv;
+
+ // Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
+ // as the intermediary directory is we can get databases recovered if
+ // crash occurred in any step of the swap. (We will recover from
+ // "safebrowsing-backup" in OpenDb().)
+ rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
+ mRootStoreDirectory, // contains old tables
+ mCacheDirectory, // common parent dir
+ mBackupDirectory); // intermediary dir for swap
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to swap in on-disk tables."));
+ RemoveUpdateIntermediaries();
+ return rv;
+ }
+
+ // Step 2. Swap in in-memory tables and update root directroy handles.
+ mLookupCaches.SwapElements(mNewLookupCaches);
+ for (auto c: mLookupCaches) {
+ c->UpdateRootDirHandle(mRootStoreDirectory);
+ }
+
+ // Step 3. Re-generate active tables based on on-disk tables.
+ rv = RegenActiveTables();
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to re-generate active tables!"));
+ }
+
+ // Step 4. Clean up intermediaries for update.
+ RemoveUpdateIntermediaries();
+
+ // Step 5. Invalidate cached tableRequest request.
+ mIsTableRequestResultOutdated = true;
+
+ LOG(("Done swap in updated tables."));
+
+ return rv;
+}
+
nsresult
Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
{
+ // Will run on the update thread after Bug 1339760.
+ // No lock is required except for CopyInUseDirForUpdate() since
+ // the table lookup may lead to database removal.
+
if (!aUpdates || aUpdates->Length() == 0) {
return NS_OK;
}
nsCOMPtr<nsIUrlClassifierUtils> urlUtil =
do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
nsCString provider;
@@ -549,76 +698,72 @@ Classifier::ApplyUpdates(nsTArray<TableU
clockStart = PR_IntervalNow();
}
nsresult rv;
{
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
- LOG(("Backup before update."));
-
- rv = BackupTables();
- NS_ENSURE_SUCCESS(rv, rv);
+ {
+ // TODO: Bug 1339050. This section should be mutual exclusive to
+ // LookupCache::Open() and HashStore::Open() since those operations
+ // might fail and cause the database to be removed. It'll be fine
+ // until we move "apply update" code off the worker thread.
+ rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to copy in-use directory for update."));
+ return rv;
+ }
+ CopyInUseLookupCacheForUpdate(); // i.e. mNewLookupCaches will be setup.
+ }
LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length()));
for (uint32_t i = 0; i < aUpdates->Length(); i++) {
// Previous UpdateHashStore() may have consumed this update..
if ((*aUpdates)[i]) {
// Run all updates for one table
nsCString updateTable(aUpdates->ElementAt(i)->TableName());
+ // Will update the mirrored in-memory and on-disk databases.
if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) {
rv = UpdateHashStore(aUpdates, updateTable);
} else {
rv = UpdateTableV4(aUpdates, updateTable);
}
- // We mark the table associated info outdated no matter the
- // update is successful to avoid any possibile non-atomic update.
- mIsTableRequestResultOutdated = true;
-
if (NS_FAILED(rv)) {
if (rv != NS_ERROR_OUT_OF_MEMORY) {
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
DumpFailedUpdate();
#endif
+ // TODO: Bug 1339050 - Should be run on worker thread.
AbortUpdateAndReset(updateTable);
}
+ RemoveUpdateIntermediaries();
return rv;
}
}
}
} // End of scopedUpdatesClearer scope.
- rv = RegenActiveTables();
- NS_ENSURE_SUCCESS(rv, rv);
-
- LOG(("Cleaning up backups."));
-
- // Move the backup directory away (signaling the transaction finished
- // successfully). This is atomic.
- rv = RemoveBackupTables();
- NS_ENSURE_SUCCESS(rv, rv);
-
- // Do the actual deletion of the backup files.
- rv = CleanToDelete();
- NS_ENSURE_SUCCESS(rv, rv);
-
- LOG(("Done applying updates."));
+ rv = SwapInNewTablesAndCleanup();
+ if (NS_FAILED(rv)) {
+ LOG(("Failed to swap in new tables."));
+ }
if (LOG_ENABLED()) {
PRIntervalTime clockEnd = PR_IntervalNow();
LOG(("update took %dms\n",
PR_IntervalToMilliseconds(clockEnd - clockStart)));
}
- return NS_OK;
+ return rv;
}
nsresult
Classifier::ApplyFullHashes(nsTArray<TableUpdate*>* aUpdates)
{
LOG(("Applying %" PRIuSIZE " table gethashes.", aUpdates->Length()));
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
@@ -810,75 +955,67 @@ Classifier::DumpFailedUpdate()
// Remove the "failed update" directory no matter it exists or not.
// Failure is fine because the directory may not exist.
failedUpdatekDirectory->Remove(true);
nsCString failedUpdatekDirName;
nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
NS_ENSURE_SUCCESS(rv, rv);
- // Move backup to a clean "failed update" directory.
- nsCOMPtr<nsIFile> backupDirectory;
- if (NS_FAILED(mBackupDirectory->Clone(getter_AddRefs(backupDirectory))) ||
- NS_FAILED(backupDirectory->MoveToNative(nullptr, failedUpdatekDirName))) {
- LOG(("Failed to move backup to the \"failed update\" directory %s",
+ // Copy the in-use directory to a clean "failed update" directory.
+ nsCOMPtr<nsIFile> inUseDirectory;
+ if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
+ NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
+ LOG(("Failed to move in-use to the \"failed update\" directory %s",
failedUpdatekDirName.get()));
return NS_ERROR_FAILURE;
}
return rv;
}
#endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
nsresult
-Classifier::BackupTables()
+Classifier::CopyInUseDirForUpdate()
{
- // We have to work in reverse here: first move the normal directory
- // away to be the backup directory, then copy the files over
- // to the normal directory. This ensures that if we crash the backup
- // dir always has a valid, complete copy, instead of a partial one,
- // because that's the one we will copy over the normal store dir.
+ LOG(("Copy in-use directory content for update."));
- nsCString backupDirName;
- nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName);
+ // We copy everything from in-use directory to a temporary directory
+ // for updating.
+
+ nsCString updatingDirName;
+ nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
- nsCString storeDirName;
- rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
- NS_ENSURE_SUCCESS(rv, rv);
-
- rv = mRootStoreDirectory->MoveToNative(nullptr, backupDirName);
- NS_ENSURE_SUCCESS(rv, rv);
-
- rv = mRootStoreDirectory->CopyToNative(nullptr, storeDirName);
+ // Remove the destination directory first (just in case) the do the copy.
+ mUpdatingDirectory->Remove(true);
+ rv = mRootStoreDirectory->CopyToNative(nullptr, updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
// We moved some things to new places, so move the handles around, too.
rv = SetupPathNames();
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
-nsresult
-Classifier::RemoveBackupTables()
+void
+Classifier::CopyInUseLookupCacheForUpdate()
{
- nsCString toDeleteName;
- nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName);
- NS_ENSURE_SUCCESS(rv, rv);
+ MOZ_ASSERT(mNewLookupCaches.IsEmpty(), "Update intermediaries is forgotten to "
+ "be removed in the previous update.");
- rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName);
- NS_ENSURE_SUCCESS(rv, rv);
-
- // mBackupDirectory now points to toDelete, fix that up.
- rv = SetupPathNames();
- NS_ENSURE_SUCCESS(rv, rv);
-
- return NS_OK;
+ // TODO: Bug 1341183 - Implement deep copy function or copy ctor.
+ // For now we just re-open every table we have opened.
+ // Another option is to NOT pre-open these LookupCaches and
+ // do a "merge" instead of "swap" after update.
+ for (auto c: mLookupCaches) {
+ Unused << GetLookupCacheForUpdate(c->TableName());
+ }
}
nsresult
Classifier::RecoverBackups()
{
bool backupExists;
nsresult rv = mBackupDirectory->Exists(&backupExists);
NS_ENSURE_SUCCESS(rv, rv);
@@ -957,30 +1094,30 @@ Classifier::UpdateHashStore(nsTArray<Tab
const nsACString& aTable)
{
if (nsUrlClassifierDBService::ShutdownHasStarted()) {
return NS_ERROR_ABORT;
}
LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
- HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory);
+ HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
if (!CheckValidUpdate(aUpdates, store.TableName())) {
return NS_OK;
}
nsresult rv = store.Open();
NS_ENSURE_SUCCESS(rv, rv);
rv = store.BeginUpdate();
NS_ENSURE_SUCCESS(rv, rv);
// Read the part of the store that is (only) in the cache
LookupCacheV2* lookupCache =
- LookupCache::Cast<LookupCacheV2>(GetLookupCache(store.TableName()));
+ LookupCache::Cast<LookupCacheV2>(GetLookupCacheForUpdate(store.TableName()));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
// Clear cache when update
lookupCache->ClearCache();
FallibleTArray<uint32_t> AddPrefixHashes;
@@ -1064,17 +1201,17 @@ Classifier::UpdateTableV4(nsTArray<Table
LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get()));
if (!CheckValidUpdate(aUpdates, aTable)) {
return NS_OK;
}
LookupCacheV4* lookupCache =
- LookupCache::Cast<LookupCacheV4>(GetLookupCache(aTable));
+ LookupCache::Cast<LookupCacheV4>(GetLookupCacheForUpdate(aTable));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
nsresult rv = NS_OK;
// If there are multiple updates for the same table, prefixes1 & prefixes2
// will act as input and output in turn to reduce memory copy overhead.
@@ -1170,47 +1307,58 @@ Classifier::UpdateCache(TableUpdate* aUp
#if defined(DEBUG)
lookupCache->DumpCache();
#endif
return NS_OK;
}
LookupCache *
-Classifier::GetLookupCache(const nsACString& aTable)
+Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
{
- for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
- if (mLookupCaches[i]->TableName().Equals(aTable)) {
- return mLookupCaches[i];
+ if (aForUpdate) {
+ return GetLookupCacheFrom(aTable, mNewLookupCaches, mUpdatingDirectory);
+ }
+ return GetLookupCacheFrom(aTable, mLookupCaches, mRootStoreDirectory);
+}
+
+LookupCache *
+Classifier::GetLookupCacheFrom(const nsACString& aTable,
+ nsTArray<LookupCache*>& aLookupCaches,
+ nsIFile* aRootStoreDirectory)
+{
+ for (uint32_t i = 0; i < aLookupCaches.Length(); i++) {
+ if (aLookupCaches[i]->TableName().Equals(aTable)) {
+ return aLookupCaches[i];
}
}
// TODO : Bug 1302600, It would be better if we have a more general non-main
// thread method to convert table name to protocol version. Currently
// we can only know this by checking if the table name ends with '-proto'.
UniquePtr<LookupCache> cache;
nsCString provider = GetProvider(aTable);
if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
- cache = MakeUnique<LookupCacheV4>(aTable, provider, mRootStoreDirectory);
+ cache = MakeUnique<LookupCacheV4>(aTable, provider, aRootStoreDirectory);
} else {
- cache = MakeUnique<LookupCacheV2>(aTable, provider, mRootStoreDirectory);
+ cache = MakeUnique<LookupCacheV2>(aTable, provider, aRootStoreDirectory);
}
nsresult rv = cache->Init();
if (NS_FAILED(rv)) {
return nullptr;
}
rv = cache->Open();
if (NS_FAILED(rv)) {
if (rv == NS_ERROR_FILE_CORRUPTED) {
Reset();
}
return nullptr;
}
- mLookupCaches.AppendElement(cache.get());
+ aLookupCaches.AppendElement(cache.get());
return cache.release();
}
nsresult
Classifier::ReadNoiseEntries(const Prefix& aPrefix,
const nsACString& aTableName,
uint32_t aCount,
PrefixArray* aNoiseEntries)