Bug 1377477 - Modify getHSTSPreloadList.js and genHPKPStaticPins.js to generate reports on what data resulted in the generated file (r?keeler) draft
authorMark Goodwin <mgoodwin@mozilla.com>
Fri, 30 Jun 2017 19:24:38 +0100
changeset 602774 8cb72f2753c9609f9f0f602854e81616f34bd151
parent 602698 0b5603017c25e943ba3ab97cb46d88adf1e6a3e4
child 635711 e208391523733ce1fe115ec5ad3606ce73fcaf45
push id66549
push usermgoodwin@mozilla.com
push dateFri, 30 Jun 2017 18:30:55 +0000
reviewerskeeler
bugs1377477
milestone56.0a1
Bug 1377477 - Modify getHSTSPreloadList.js and genHPKPStaticPins.js to generate reports on what data resulted in the generated file (r?keeler) MozReview-Commit-ID: I1KxbEcvjVF
security/manager/tools/genHPKPStaticPins.js
security/manager/tools/getHSTSPreloadList.js
--- a/security/manager/tools/genHPKPStaticPins.js
+++ b/security/manager/tools/genHPKPStaticPins.js
@@ -4,24 +4,28 @@
 
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/genHPKPStaticpins.js \
 //                                  [absolute path to]/PreloadedHPKPins.json \
 //                                  [an unused argument - see bug 1205406] \
-//                                  [absolute path to]/StaticHPKPins.h
+//                                  [absolute path to]/StaticHPKPins.h \
+//                                  [absolute path to]/StaticHPKPData.json \
+//                                  [absolute path to]/version.txt
 "use strict";
 
-if (arguments.length != 3) {
+if (arguments.length < 3) {
   throw new Error("Usage: genHPKPStaticPins.js " +
                   "<absolute path to PreloadedHPKPins.json> " +
                   "<an unused argument - see bug 1205406> " +
-                  "<absolute path to StaticHPKPins.h>");
+                  "<absolute path to StaticHPKPins.h>" +
+                  "(optional) <absolute path to StaticHPKPData.json>" +
+                  "(optional) <absolute path to version.txt");
 }
 
 const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
 
 var { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
 var { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
 var { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 
@@ -68,16 +72,37 @@ var gStaticPins = parseJson(arguments[0]
 
 // arguments[1] is ignored for now. See bug 1205406.
 
 // Open the output file.
 var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
 file.initWithPath(arguments[2]);
 var gFileOutputStream = FileUtils.openSafeFileOutputStream(file);
 
+let gJsonFileOutputStream;
+if (arguments.length > 3) {
+  var jsonFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+  jsonFile.initWithPath(arguments[3]);
+  gJsonFileOutputStream = FileUtils.openSafeFileOutputStream(jsonFile);
+}
+
+let gVersion = "unknown";
+if (arguments.length > 4) {
+  gVersion = readFileToString(arguments[4]).trim();
+}
+
+let gJSONReportData = {"fingerprints": [],
+  "pinsets": [],
+  "domainlists": [],
+  "creationInfo": {
+    "version": gVersion,
+    "date": (new Date()).toISOString()
+  }};
+
+
 function writeString(string) {
   gFileOutputStream.write(string, string.length);
 }
 
 function readFileToString(filename) {
   let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
   file.initWithPath(filename);
   let stream = Cc["@mozilla.org/network/file-input-stream;1"]
@@ -522,16 +547,17 @@ function writeDomainList(chromeImportedE
       chromeImportedEntries.splice(i, 1);
     }
   }
   let sortedEntries = gStaticPins.entries;
   sortedEntries.push.apply(sortedEntries, chromeImportedEntries);
   for (let entry of sortedEntries.sort(compareByName)) {
     count++;
     writeEntry(entry);
+    gJSONReportData.domainlists.push(entry);
   }
   writeString("};\n");
 
   writeString("\n// Pinning Preload List Length = " + count + ";\n");
   writeString("\nstatic const int32_t kUnknownId = -1;\n");
 }
 
 function writeFile(certNameToSKD, certSKDToName,
@@ -557,40 +583,48 @@ function writeFile(certNameToSKD, certSK
 
   // Write actual fingerprints.
   Object.keys(usedFingerprints).sort().forEach(function(certName) {
     if (certName) {
       writeString("/* " + certName + " */\n");
       writeString("static const char " + nameToAlias(certName) + "[] =\n");
       writeString("  \"" + certNameToSKD[certName] + "\";\n");
       writeString("\n");
+      gJSONReportData.fingerprints.push({"name": certName, "skd": certNameToSKD[certName]});
     }
   });
 
   // Write the pinsets
   writeString(PINSETDEF);
   writeString("/* PreloadedHPKPins.json pinsets */\n");
   gStaticPins.pinsets.sort(compareByName).forEach(function(pinset) {
     writeFullPinset(certNameToSKD, certSKDToName, pinset);
+    gJSONReportData.pinsets.push({"source": "PreloadedHPKPins.json pinset", "name": pinset.name, "hashes": pinset.sha256_hashes});
   });
   writeString("/* Chrome static pinsets */\n");
   for (let key in chromeImportedPinsets) {
     if (mozillaPins[key]) {
       dump("Skipping duplicate pinset " + key + "\n");
     } else {
       dump("Writing pinset " + key + "\n");
       writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]);
+      gJSONReportData.pinsets.push({"source": "Chrome static pinset", "name": chromeImportedPinsets[key].name, "hashes": chromeImportedPinsets[key].sha256_hashes});
     }
   }
 
   // Write the domainlist entries.
   writeString(DOMAINHEADER);
   writeDomainList(chromeImportedEntries);
   writeString("\n");
   writeString(genExpirationTime());
+
+  if (gJsonFileOutputStream) {
+    let reportString = JSON.stringify(gJSONReportData);
+    gJsonFileOutputStream.write(reportString, reportString.length);
+  }
 }
 
 function loadExtraCertificates(certStringList) {
   let constructedCerts = [];
   for (let certString of certStringList) {
     constructedCerts.push(gCertDB.constructX509FromBase64(certString));
   }
   return constructedCerts;
@@ -603,8 +637,11 @@ var [ chromeNameToHash, chromeNameToMozN
 var [ chromeImportedPinsets, chromeImportedEntries ] =
   downloadAndParseChromePins(gStaticPins.chromium_data.json_file_url,
     chromeNameToHash, chromeNameToMozName, certNameToSKD, certSKDToName);
 
 writeFile(certNameToSKD, certSKDToName, chromeImportedPinsets,
           chromeImportedEntries);
 
 FileUtils.closeSafeFileOutputStream(gFileOutputStream);
+if (gJsonFileOutputStream) {
+  FileUtils.closeSafeFileOutputStream(gJsonFileOutputStream);
+}
--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -3,19 +3,21 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/getHSTSPreloadlist.js \
-//                                  [absolute path to]/nsSTSPreloadlist.inc'
+//                                  [absolute path to]/nsSTSPreloadlist.inc' \
+//                                  [absolute path to]/StaticHSTSData.json
 // Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
-//       working directory.
+//       working directory and creates a JSON file at the path specified for
+//       StaticHSTSData.json
 
 var Cc = Components.classes;
 var Ci = Components.interfaces;
 var Cu = Components.utils;
 var Cr = Components.results;
 
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/FileUtils.jsm");
@@ -38,16 +40,18 @@ const HEADER = "/* This Source Code Form
 "\n" +
 "/*****************************************************************************/\n" +
 "/* This is an automatically generated file. If you're not                    */\n" +
 "/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
 
+var gFilename;
+
 function download() {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
             .createInstance(Ci.nsIXMLHttpRequest);
   req.open("GET", SOURCE, false); // doing the request synchronously
   try {
     req.send();
   } catch (e) {
     throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
@@ -220,45 +224,65 @@ function compareHSTSStatus(a, b) {
 
 function writeTo(string, fos) {
   fos.write(string, string.length);
 }
 
 // Determines and returns a string representing a declaration of when this
 // preload list should no longer be used.
 // This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
-function getExpirationTimeString() {
+function getExpirationTime() {
   var now = new Date();
   var nowMillis = now.getTime();
   // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
   var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
-  var expirationMicros = expirationMillis * 1000;
+  return expirationMillis * 1000;
+}
+
+function getExpirationTimeString(expirationMicros) {
   return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
 }
 
 function errorToString(status) {
   return (status.error == ERROR_MAX_AGE_TOO_LOW
           ? status.error + status.maxAge
           : status.error);
 }
 
 function writeEntry(status, indices, outputStream) {
   let includeSubdomains = (status.finalIncludeSubdomains ? "true" : "false");
   writeTo("  { " + indices[status.name] + ", " + includeSubdomains + " },\n",
           outputStream);
 }
 
 function output(sortedStatuses, currentList) {
+
   try {
     var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
     var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
     var fos = FileUtils.openSafeFileOutputStream(file);
     var eos = FileUtils.openSafeFileOutputStream(errorFile);
     writeTo(HEADER, fos);
-    writeTo(getExpirationTimeString(), fos);
+    var expirationTime = getExpirationTime();
+    writeTo(getExpirationTimeString(expirationTime), fos);
+
+    let reportData = {
+      "statuses":[],
+      "creationInfo": {
+        "date": (new Date()).toISOString(),
+        "expiration": expirationTime
+      }
+    };
+
+    let jsonFileOutputStream;
+    if (gFilename) {
+      var jsonFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+      jsonFile.initWithPath(gFilename);
+      jsonFileOutputStream = FileUtils.openSafeFileOutputStream(jsonFile);
+    }
 
     for (let status in sortedStatuses) {
       // If we've encountered an error for this entry (other than the site not
       // sending an HSTS header), be safe and don't remove it from the list
       // (given that it was already on the list).
       if (status.error != ERROR_NONE &&
           status.error != ERROR_NO_HSTS_HEADER &&
           status.error != ERROR_MAX_AGE_TOO_LOW &&
@@ -306,16 +330,17 @@ function output(sortedStatuses, currentL
       // Add 1 for the null terminator in C.
       currentIndex += status.name.length + 1;
       // Rebuilding the preload list requires reading the previous preload
       // list.  Write out a comment describing each host prior to writing out
       // the string for the host.
       writeTo("  /* \"" + status.name + "\", " +
               (status.finalIncludeSubdomains ? "true" : "false") + " */ ",
               fos);
+      reportData.statuses.push({"name": status.name, "includeSubdomains": status.finalIncludeSubdomains});
       // Write out the string itself as individual characters, including the
       // null terminator.  We do it this way rather than using C's string
       // concatentation because some compilers have hardcoded limits on the
       // lengths of string literals, and the preload list is large enough
       // that it runs into said limits.
       for (let c of status.name) {
         writeTo("'" + c + "', ", fos);
       }
@@ -336,16 +361,21 @@ function output(sortedStatuses, currentL
 
     writeTo(PREFIX, fos);
     for (let status of includedStatuses) {
       writeEntry(status, indices, fos);
     }
     writeTo(POSTFIX, fos);
     FileUtils.closeSafeFileOutputStream(fos);
     FileUtils.closeSafeFileOutputStream(eos);
+
+    if (jsonFileOutputStream) {
+      writeTo(JSON.stringify(reportData), jsonFileOutputStream);
+      FileUtils.closeSafeFileOutputStream(jsonFileOutputStream);
+    }
   } catch (e) {
     dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
   }
 }
 
 function shouldRetry(response) {
   return (response.error != ERROR_NO_HSTS_HEADER &&
           response.error != ERROR_MAX_AGE_TOO_LOW &&
@@ -455,22 +485,27 @@ function insertTestHosts(hstsStatuses) {
       forceInclude: true,
       originalIncludeSubdomains: testEntry.includeSubdomains,
     });
   }
 }
 
 // ****************************************************************************
 // This is where the action happens:
-if (arguments.length != 1) {
+if (arguments.length < 1) {
   throw new Error("Usage: getHSTSPreloadList.js " +
-                  "<absolute path to current nsSTSPreloadList.inc>");
+                  "<absolute path to current nsSTSPreloadList.inc> " +
+                  "<absolute path to StaticHSTSData.json>");
 }
 // get the current preload list
 var currentHosts = readCurrentList(arguments[0]);
+// get the filename for the JSON report
+if (arguments.length > 1) {
+  gFilename = arguments[1];
+}
 // delete any hosts we use in tests so we don't actually connect to them
 deleteTestHosts(currentHosts);
 // disable the current preload list so it won't interfere with requests we make
 Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
 // download and parse the raw json file from the Chromium source
 var rawdata = download();
 // get just the hosts with mode: "force-https"
 var hosts = getHosts(rawdata);