Bug 1434300 - Add a utility to match certificates based on SPKI r?keeler r?fkiefer draft
authorJ.C. Jones <jjones@mozilla.com>
Wed, 21 Feb 2018 14:08:44 -0500
changeset 758022 6695c4b58a9769a22fae73965a3aec27771e09f8
parent 758021 c1e4910567cf84ca810e0cef63dd5cf1e0c9dd79
child 758023 e57ffb2989337f2b601e5153eb45f839cf27c7be
push id99916
push userbmo:jjones@mozilla.com
push dateWed, 21 Feb 2018 19:10:27 +0000
reviewerskeeler, fkiefer
bugs1434300
milestone60.0a1
Bug 1434300 - Add a utility to match certificates based on SPKI r?keeler r?fkiefer This modifies crtshToDNStruct.py to be able to produce SPKI or DN-based lists, and adds a SPKI-search method to TrustOverrideUtils.h. This also regenerates the TrustOverride files to use the new script. MozReview-Commit-ID: BhMoJbYXs7Y
security/certverifier/TrustOverride-AppleGoogleData.inc
security/certverifier/TrustOverride-SymantecData.inc
security/certverifier/TrustOverride-TestImminentDistrustData.inc
security/certverifier/TrustOverrideUtils.h
security/certverifier/tests/gtest/moz.build
security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
security/manager/tools/crtshToDNStruct/requirements.txt
security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
security/manager/tools/crtshToIdentifyingStruct/requirements.txt
--- a/security/certverifier/TrustOverride-AppleGoogleData.inc
+++ b/security/certverifier/TrustOverride-AppleGoogleData.inc
@@ -1,10 +1,10 @@
-// Script from security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
-// Invocation: crtshToDNStruct.py 142951186 23635000 5250464 12716200 19602712 19602724 21760447 19602706 19602741
+// Script from security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
+// Invocation: crtshToIdentifyingStruct.py -dn -listname RootAppleAndGoogleDNs 142951186 23635000 5250464 12716200 19602712 19602724 21760447 19602706 19602741
 
 // /C=US/O=Google Inc/CN=Google Internet Authority G2
 // SHA256 Fingerprint: 9B:75:9D:41:E3:DE:30:F9:D2:F9:02:02:7D:79:2B:65
 //                     D9:50:A9:8B:BB:6D:6D:56:BE:7F:25:28:45:3B:F8:E9
 // https://crt.sh/?id=142951186 (crt.sh ID=142951186)
 //
 // and
 //
@@ -122,25 +122,17 @@ static const uint8_t CAAppleISTCA6G1DN[1
   0x04, 0x0B, 0x0C, 0x17, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61,
   0x74, 0x69, 0x6F, 0x6E, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x74,
   0x79, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x0A, 0x0C, 0x0A, 0x41,
   0x70, 0x70, 0x6C, 0x65, 0x20, 0x49, 0x6E, 0x63, 0x2E, 0x31, 0x0B, 0x30, 0x09,
   0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x55, 0x53,
 };
 
 static const DataAndLength RootAppleAndGoogleDNs[]= {
-  { CAGoogleInternetAuthorityG2DN,
-    sizeof(CAGoogleInternetAuthorityG2DN) },
-  { CAAppleISTCA2G1DN,
-    sizeof(CAAppleISTCA2G1DN) },
-  { CAAppleISTCA5G1DN,
-    sizeof(CAAppleISTCA5G1DN) },
-  { CAAppleISTCA4G1DN,
-    sizeof(CAAppleISTCA4G1DN) },
-  { CAAppleISTCA7G1DN,
-    sizeof(CAAppleISTCA7G1DN) },
-  { CAAppleISTCA8G1DN,
-    sizeof(CAAppleISTCA8G1DN) },
-  { CAAppleISTCA3G1DN,
-    sizeof(CAAppleISTCA3G1DN) },
-  { CAAppleISTCA6G1DN,
-    sizeof(CAAppleISTCA6G1DN) },
+  { CAGoogleInternetAuthorityG2DN, sizeof(CAGoogleInternetAuthorityG2DN) },
+  { CAAppleISTCA2G1DN, sizeof(CAAppleISTCA2G1DN) },
+  { CAAppleISTCA5G1DN, sizeof(CAAppleISTCA5G1DN) },
+  { CAAppleISTCA4G1DN, sizeof(CAAppleISTCA4G1DN) },
+  { CAAppleISTCA7G1DN, sizeof(CAAppleISTCA7G1DN) },
+  { CAAppleISTCA8G1DN, sizeof(CAAppleISTCA8G1DN) },
+  { CAAppleISTCA3G1DN, sizeof(CAAppleISTCA3G1DN) },
+  { CAAppleISTCA6G1DN, sizeof(CAAppleISTCA6G1DN) },
 };
--- a/security/certverifier/TrustOverride-SymantecData.inc
+++ b/security/certverifier/TrustOverride-SymantecData.inc
@@ -1,10 +1,10 @@
-// Script from security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
-// Invocation: crtshToDNStruct.py 17 3381895 847444 4350 4174851 4175126 12729019 8983600 12726040 8983601 30 3382830 254193 8984570 68409 26682 2771491 93 1039083
+// Script from security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
+// Invocation: crtshToIdentifyingStruct.py -dn -listname RootSymantecDNs 17 3381895 847444 4350 4174851 4175126 12729019 8983600 12726040 8983601 30 3382830 254193 8984570 68409 26682 2771491 93 1039083
 
 // /C=US/O=GeoTrust Inc./CN=GeoTrust Global CA
 // SHA256 Fingerprint: FF:85:6A:2D:25:1D:CD:88:D3:66:56:F4:50:12:67:98
 //                     CF:AB:AA:DE:40:79:9C:72:2D:E4:D2:B5:DB:36:A7:3A
 // https://crt.sh/?id=17 (crt.sh ID=17)
 static const uint8_t CAGeoTrustGlobalCADN[68] = {
   0x30, 0x42, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02,
   0x55, 0x53, 0x31, 0x16, 0x30, 0x14, 0x06, 0x03, 0x55, 0x04, 0x0A, 0x13, 0x0D,
@@ -361,42 +361,36 @@ static const uint8_t CAVeriSignUniversal
   0x6F, 0x6E, 0x6C, 0x79, 0x31, 0x38, 0x30, 0x36, 0x06, 0x03, 0x55, 0x04, 0x03,
   0x13, 0x2F, 0x56, 0x65, 0x72, 0x69, 0x53, 0x69, 0x67, 0x6E, 0x20, 0x55, 0x6E,
   0x69, 0x76, 0x65, 0x72, 0x73, 0x61, 0x6C, 0x20, 0x52, 0x6F, 0x6F, 0x74, 0x20,
   0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E,
   0x20, 0x41, 0x75, 0x74, 0x68, 0x6F, 0x72, 0x69, 0x74, 0x79,
 };
 
 static const DataAndLength RootSymantecDNs[]= {
-  { CAGeoTrustGlobalCADN,
-    sizeof(CAGeoTrustGlobalCADN) },
+  { CAGeoTrustGlobalCADN, sizeof(CAGeoTrustGlobalCADN) },
   { CAGeoTrustPrimaryCertificationAuthorityG2DN,
     sizeof(CAGeoTrustPrimaryCertificationAuthorityG2DN) },
   { CAGeoTrustPrimaryCertificationAuthorityG3DN,
     sizeof(CAGeoTrustPrimaryCertificationAuthorityG3DN) },
   { CAGeoTrustPrimaryCertificationAuthorityDN,
     sizeof(CAGeoTrustPrimaryCertificationAuthorityDN) },
-  { CAGeoTrustUniversalCADN,
-    sizeof(CAGeoTrustUniversalCADN) },
-  { CAGeoTrustUniversalCA2DN,
-    sizeof(CAGeoTrustUniversalCA2DN) },
+  { CAGeoTrustUniversalCADN, sizeof(CAGeoTrustUniversalCADN) },
+  { CAGeoTrustUniversalCA2DN, sizeof(CAGeoTrustUniversalCA2DN) },
   { CASymantecClass1PublicPrimaryCertificationAuthorityG4DN,
     sizeof(CASymantecClass1PublicPrimaryCertificationAuthorityG4DN) },
   { CASymantecClass1PublicPrimaryCertificationAuthorityG6DN,
     sizeof(CASymantecClass1PublicPrimaryCertificationAuthorityG6DN) },
   { CASymantecClass2PublicPrimaryCertificationAuthorityG4DN,
     sizeof(CASymantecClass2PublicPrimaryCertificationAuthorityG4DN) },
   { CASymantecClass2PublicPrimaryCertificationAuthorityG6DN,
     sizeof(CASymantecClass2PublicPrimaryCertificationAuthorityG6DN) },
-  { CAthawtePrimaryRootCADN,
-    sizeof(CAthawtePrimaryRootCADN) },
-  { CAthawtePrimaryRootCAG2DN,
-    sizeof(CAthawtePrimaryRootCAG2DN) },
-  { CAthawtePrimaryRootCAG3DN,
-    sizeof(CAthawtePrimaryRootCAG3DN) },
+  { CAthawtePrimaryRootCADN, sizeof(CAthawtePrimaryRootCADN) },
+  { CAthawtePrimaryRootCAG2DN, sizeof(CAthawtePrimaryRootCAG2DN) },
+  { CAthawtePrimaryRootCAG3DN, sizeof(CAthawtePrimaryRootCAG3DN) },
   { CAVeriSignClass1PublicPrimaryCertificationAuthorityG3DN,
     sizeof(CAVeriSignClass1PublicPrimaryCertificationAuthorityG3DN) },
   { CAVeriSignClass2PublicPrimaryCertificationAuthorityG3DN,
     sizeof(CAVeriSignClass2PublicPrimaryCertificationAuthorityG3DN) },
   { CAVeriSignClass3PublicPrimaryCertificationAuthorityG3DN,
     sizeof(CAVeriSignClass3PublicPrimaryCertificationAuthorityG3DN) },
   { CAVeriSignClass3PublicPrimaryCertificationAuthorityG4DN,
     sizeof(CAVeriSignClass3PublicPrimaryCertificationAuthorityG4DN) },
--- a/security/certverifier/TrustOverride-TestImminentDistrustData.inc
+++ b/security/certverifier/TrustOverride-TestImminentDistrustData.inc
@@ -1,10 +1,10 @@
-// Script from security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
-// Invocation: security/manager/tools/crtshToDNStruct/crtshToDNStruct.py security/manager/ssl/tests/unit/bad_certs/ee-imminently-distrusted.pem
+// Script from security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
+// Invocation: crtshToIdentifyingStruct.py -dn -listname TestImminentDistrustEndEntityDNs ../../ssl/tests/unit/bad_certs/ee-imminently-distrusted.pem
 
 // This file is used by test_imminent_distrust.js and by
 // browser_console_certificate_imminent_distrust.js to ensure that the UI for
 // alerting users to an upcoming CA distrust action continues to function.
 
 // /C=US/CN=Imminently Distrusted End Entity
 // SHA256 Fingerprint: 63:3A:70:8A:67:42:91:95:98:E9:D1:CB:8B:5D:73:80
 //                     BA:6D:AD:25:82:62:52:AD:5E:5E:DC:06:BF:03:1F:D0
--- a/security/certverifier/TrustOverrideUtils.h
+++ b/security/certverifier/TrustOverrideUtils.h
@@ -22,25 +22,43 @@ template<size_t T>
 static bool
 CertDNIsInList(const CERTCertificate* aCert, const DataAndLength (&aDnList)[T])
 {
   MOZ_ASSERT(aCert);
   if (!aCert) {
     return false;
   }
 
-  for (auto &dn: aDnList) {
+  for (auto& dn: aDnList) {
     if (aCert->derSubject.len == dn.len &&
         mozilla::PodEqual(aCert->derSubject.data, dn.data, dn.len)) {
       return true;
     }
   }
   return false;
 }
 
+template<size_t T>
+static bool
+CertSPKIIsInList(const CERTCertificate* aCert, const DataAndLength (&aSpkiList)[T])
+{
+  MOZ_ASSERT(aCert);
+  if (!aCert) {
+    return false;
+  }
+
+  for (auto& spki: aSpkiList) {
+    if (aCert->derPublicKey.len == spki.len &&
+        mozilla::PodEqual(aCert->derPublicKey.data, spki.data, spki.len)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 template<size_t T, size_t R>
 static bool
 CertMatchesStaticData(const CERTCertificate* cert,
                       const unsigned char (&subject)[T],
                       const unsigned char (&spki)[R]) {
   MOZ_ASSERT(cert);
   if (!cert) {
     return false;
--- a/security/certverifier/tests/gtest/moz.build
+++ b/security/certverifier/tests/gtest/moz.build
@@ -12,13 +12,14 @@ SOURCES += [
     'CTPolicyEnforcerTest.cpp',
     'CTSerializationTest.cpp',
     'CTTestUtils.cpp',
     'MultiLogCTVerifierTest.cpp',
 ]
 
 LOCAL_INCLUDES += [
     '/security/certverifier',
+    '/security/manager/ssl',
     '/security/pkix/include',
     '/security/pkix/lib',
 ]
 
 FINAL_LIBRARY = 'xul-gtest'
rename from security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
rename to security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
--- a/security/manager/tools/crtshToDNStruct/crtshToDNStruct.py
+++ b/security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
@@ -1,22 +1,23 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
 This utility takes a series of https://crt.sh/ identifiers and writes to
-stdout all of those certs' distinguished name fields in hex, with an array
-of all those named "RootDNs". You'll need to post-process this list to rename
-"RootDNs" and handle any duplicates.
+stdout all of those certs' distinguished name or SPKI fields in hex, with an
+array of all those. You'll need to post-process this list to handle any
+duplicates.
 
 Requires Python 3.
 """
+import argparse
 import re
 import requests
 import sys
 import io
 
 from pyasn1.codec.der import decoder
 from pyasn1.codec.der import encoder
 from pyasn1_modules import pem
@@ -43,25 +44,33 @@ def nameOIDtoString(oid):
     if oid == NameOID.LOCALITY_NAME:
         return "L"
     if oid == NameOID.ORGANIZATION_NAME:
         return "O"
     if oid == NameOID.ORGANIZATIONAL_UNIT_NAME:
         return "OU"
     raise Exception("Unknown OID: {}".format(oid))
 
-def print_block(pemData, crtshId):
+def print_block(pemData, identifierType="DN", crtshId=None):
     substrate = pem.readPemFromFile(io.StringIO(pemData.decode("utf-8")))
     cert, rest = decoder.decode(substrate, asn1Spec=rfc5280.Certificate())
-    der_subject = encoder.encode(cert['tbsCertificate']['subject'])
-    octets = hex_string_for_struct(der_subject)
+    octets = None
+
+    if identifierType == "DN":
+        der_subject = encoder.encode(cert['tbsCertificate']['subject'])
+        octets = hex_string_for_struct(der_subject)
+    elif identifierType == "SPKI":
+        der_spki = encoder.encode(cert['tbsCertificate']['subjectPublicKeyInfo'])
+        octets = hex_string_for_struct(der_spki)
+    else:
+        raise Exception("Unknown identifier type: " + identifierType)
 
     cert = x509.load_pem_x509_certificate(pemData, default_backend())
     common_name = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0]
-    block_name = "CA{}DN".format(re.sub(r'[-:=_. ]', '', common_name.value))
+    block_name = "CA{}{}".format(re.sub(r'[-:=_. ]', '', common_name.value), identifierType)
 
     fingerprint = hex_string_human_readable(cert.fingerprint(hashes.SHA256()))
 
     dn_parts = ["/{id}={value}".format(id=nameOIDtoString(part.oid),
                                        value=part.value) for part in cert.subject]
     distinguished_name = "".join(dn_parts)
 
     print("// {dn}".format(dn=distinguished_name))
@@ -78,29 +87,54 @@ def print_block(pemData, crtshId):
 
     print("};")
     print()
 
     return block_name
 
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-spki", action="store_true",
+                        help="Create a list of subject public key info fields")
+    parser.add_argument("-dn", action="store_true",
+                        help="Create a list of subject distinguished name fields")
+    parser.add_argument("-listname",
+                        help="Name of the final DataAndLength block")
+    parser.add_argument("certId", nargs="+",
+                        help="A list of PEM files on disk or crt.sh IDs")
+    args = parser.parse_args()
+
+    if not args.dn and not args.spki:
+        parser.print_help()
+        raise Exception("You must select either DN or SPKI matching")
+
     blocks = []
 
-    certshIds = sys.argv[1:]
-    print("// Script from security/manager/tools/crtshToDNStruct/crtshToDNStruct.py")
-    print("// Invocation: {} {}".format(sys.argv[0], " ".join(certshIds)))
+    print("// Script from security/manager/tools/crtshToIdentifyingStruct/" +
+          "crtshToIdentifyingStruct.py")
+    print("// Invocation: {}".format(" ".join(sys.argv)))
     print()
-    for crtshId in certshIds:
+
+    identifierType = None
+    if args.dn:
+        identifierType = "DN"
+    else:
+        identifierType = "SPKI"
+
+    for certId in args.certId:
         # Try a local file first, then crt.sh
         try:
-            with open(crtshId, "rb") as pemFile:
-                blocks.append(print_block(pemFile.read(), None))
-        except FileNotFoundError:
-            r = requests.get('https://crt.sh/?d={}'.format(crtshId))
+            with open(certId, "rb") as pemFile:
+                blocks.append(print_block(pemFile.read(), identifierType=identifierType))
+        except OSError:
+            r = requests.get('https://crt.sh/?d={}'.format(certId))
             r.raise_for_status()
-            blocks.append(print_block(r.content, crtshId))
+            blocks.append(print_block(r.content, crtshId=certId, identifierType=identifierType))
 
-    print("static const DataAndLength RootDNs[]= {")
+    print("static const DataAndLength " + args.listname + "[]= {")
     for structName in blocks:
-        print("  { " + "{},".format(structName))
-        print("    sizeof({})".format(structName) + " },")
+        if len(structName) < 33:
+            print("  { " + "{name}, sizeof({name}) ".format(name=structName) + "},")
+        else:
+            print("  { " + "{},".format(structName))
+            print("    sizeof({})".format(structName) + " },")
     print("};")
rename from security/manager/tools/crtshToDNStruct/requirements.txt
rename to security/manager/tools/crtshToIdentifyingStruct/requirements.txt