bug 1396137 - update broken fips pkcs#11 module db handling code for when we use the sqlite-backed databses r?jcj draft
authorDavid Keeler <dkeeler@mozilla.com>
Fri, 01 Sep 2017 15:54:40 -0700
changeset 659165 cabfaf23bfe23ce6db00012900b57ee3126bf5bd
parent 659065 3ecda4678c49ca255c38b1697142b9118cdd27e7
child 729913 0031d1d2456880b903edd0bc2b7f5cb218004eab
push id78046
push userbmo:dkeeler@mozilla.com
push dateTue, 05 Sep 2017 17:12:33 +0000
reviewersjcj
bugs1396137
milestone57.0a1
bug 1396137 - update broken fips pkcs#11 module db handling code for when we use the sqlite-backed databses r?jcj This handles the different error code returned by NSS and that the pkcs#11 module db has a different filename. MozReview-Commit-ID: HJK4zsf6IS0
security/manager/ssl/nsNSSComponent.cpp
--- a/security/manager/ssl/nsNSSComponent.cpp
+++ b/security/manager/ssl/nsNSSComponent.cpp
@@ -1763,29 +1763,30 @@ GetNSSProfilePath(nsAutoCString& aProfil
 
 #ifndef ANDROID
 // Given a profile path, attempt to rename the PKCS#11 module DB to
 // "<original name>.fips". In the case of a catastrophic failure (e.g. out of
 // memory), returns a failing nsresult. If execution could conceivably proceed,
 // returns NS_OK even if renaming the file didn't work. This simplifies the
 // logic of the calling code.
 static nsresult
-AttemptToRenamePKCS11ModuleDB(const nsACString& profilePath)
+AttemptToRenamePKCS11ModuleDB(const nsACString& profilePath,
+                              const nsACString& moduleDBFilename)
 {
   // profilePath may come from the environment variable
   // MOZPSM_NSSDBDIR_OVERRIDE. If so, the user's NSS DBs are most likely not in
   // their profile directory and we shouldn't mess with them.
   const char* dbDirOverride = getenv("MOZPSM_NSSDBDIR_OVERRIDE");
   if (dbDirOverride && strlen(dbDirOverride) > 0) {
     MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
             ("MOZPSM_NSSDBDIR_OVERRIDE set - not renaming PKCS#11 module DB"));
     return NS_OK;
   }
-  NS_NAMED_LITERAL_CSTRING(moduleDBFilename, "secmod.db");
-  NS_NAMED_LITERAL_CSTRING(destModuleDBFilename, "secmod.db.fips");
+  nsAutoCString destModuleDBFilename(moduleDBFilename);
+  destModuleDBFilename.Append(".fips");
   nsCOMPtr<nsIFile> dbFile = do_CreateInstance("@mozilla.org/file/local;1");
   if (!dbFile) {
     return NS_ERROR_FAILURE;
   }
   nsresult rv = dbFile->InitWithNativePath(profilePath);
   if (NS_FAILED(rv)) {
     return rv;
   }
@@ -1797,17 +1798,17 @@ AttemptToRenamePKCS11ModuleDB(const nsAC
   bool exists;
   rv = dbFile->Exists(&exists);
   if (NS_FAILED(rv)) {
     return rv;
   }
   // This is strange, but not a catastrophic failure.
   if (!exists) {
     MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
-            ("%s doesn't exist?", moduleDBFilename.get()));
+            ("%s doesn't exist?", PromiseFlatCString(moduleDBFilename).get()));
     return NS_OK;
   }
   nsCOMPtr<nsIFile> destDBFile = do_CreateInstance("@mozilla.org/file/local;1");
   if (!destDBFile) {
     return NS_ERROR_FAILURE;
   }
   rv = destDBFile->InitWithNativePath(profilePath);
   if (NS_FAILED(rv)) {
@@ -1840,16 +1841,32 @@ AttemptToRenamePKCS11ModuleDB(const nsAC
     return rv;
   }
   // This may fail on, e.g., a read-only file system. This would be unfortunate,
   // but again it isn't catastropic and we would want to fall back to
   // initializing NSS in no-DB mode.
   Unused << dbFile->MoveToNative(profileDir, destModuleDBFilename);
   return NS_OK;
 }
+
+// We may be using the legacy databases, in which case we need to use
+// "secmod.db". We may be using the sqlite-backed databases, in which case we
+// need to use "pkcs11.txt".
+static nsresult
+AttemptToRenameBothPKCS11ModuleDBVersions(const nsACString& profilePath)
+{
+  NS_NAMED_LITERAL_CSTRING(legacyModuleDBFilename, "secmod.db");
+  NS_NAMED_LITERAL_CSTRING(sqlModuleDBFilename, "pkcs11.txt");
+  nsresult rv = AttemptToRenamePKCS11ModuleDB(profilePath,
+                                              legacyModuleDBFilename);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  return AttemptToRenamePKCS11ModuleDB(profilePath, sqlModuleDBFilename);
+}
 #endif // ifndef ANDROID
 
 // Given a profile directory, attempt to initialize NSS. If nocertdb is true,
 // (or if we don't have a profile directory) simply initialize NSS in no DB mode
 // and return. Otherwise, first attempt to initialize in read/write mode, and
 // then read-only mode if that fails. If both attempts fail, we may be failing
 // to initialize an NSS DB collection that has FIPS mode enabled. Attempt to
 // ascertain if this is the case, and if so, rename the offending PKCS#11 module
@@ -1887,24 +1904,30 @@ InitializeNSSWithFallbacks(const nsACStr
   // That failed. Try read-only mode.
   srv = ::mozilla::psm::InitializeNSS(profilePathCStr.get(), true, !safeMode);
   if (srv == SECSuccess) {
     MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r-o mode"));
     return NS_OK;
   }
 #ifndef ANDROID
   savedPRErrorCode2 = PR_GetError();
+
+  MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
+          ("failed to initialize NSS with codes %d %d", savedPRErrorCode1,
+           savedPRErrorCode2));
 #endif // ifndef ANDROID
 
 #ifndef ANDROID
   // That failed as well. Maybe we're trying to load a PKCS#11 module DB that is
   // in FIPS mode, but we don't support FIPS? Test load NSS without PKCS#11
   // modules. If that succeeds, that's probably what's going on.
   if (!safeMode && (savedPRErrorCode1 == SEC_ERROR_LEGACY_DATABASE ||
-                    savedPRErrorCode2 == SEC_ERROR_LEGACY_DATABASE)) {
+                    savedPRErrorCode2 == SEC_ERROR_LEGACY_DATABASE ||
+                    savedPRErrorCode1 == SEC_ERROR_PKCS11_DEVICE_ERROR ||
+                    savedPRErrorCode2 == SEC_ERROR_PKCS11_DEVICE_ERROR)) {
     MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("attempting no-module db init"));
     // It would make sense to initialize NSS in read-only mode here since this
     // is just a test to see if the PKCS#11 module DB being in FIPS mode is the
     // problem, but for some reason the combination of read-only and no-moddb
     // flags causes NSS initialization to fail, so unfortunately we have to use
     // read-write mode.
     srv = ::mozilla::psm::InitializeNSS(profilePathCStr.get(), false, false);
     if (srv == SECSuccess) {
@@ -1913,17 +1936,17 @@ InitializeNSSWithFallbacks(const nsACStr
       srv = NSS_Shutdown();
       if (srv != SECSuccess) {
         return NS_ERROR_FAILURE;
       }
       MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("trying to rename module db"));
       // If this fails non-catastrophically, we'll attempt to initialize NSS
       // again in r/w then r-o mode (both of which will fail), and then we'll
       // fall back to NSS_NoDB_Init, which is the behavior we want.
-      nsresult rv = AttemptToRenamePKCS11ModuleDB(profilePath);
+      nsresult rv = AttemptToRenameBothPKCS11ModuleDBVersions(profilePath);
       if (NS_FAILED(rv)) {
         return rv;
       }
       srv = ::mozilla::psm::InitializeNSS(profilePathCStr.get(), false, true);
       if (srv == SECSuccess) {
         MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r/w mode"));
         return NS_OK;
       }