Bug 1377477 - Modify getHSTSPreloadList.js and genHPKPStaticPins.js to generate reports on what data resulted in the generated file (r?keeler)
MozReview-Commit-ID: I1KxbEcvjVF
--- a/security/manager/tools/genHPKPStaticPins.js
+++ b/security/manager/tools/genHPKPStaticPins.js
@@ -4,24 +4,28 @@
// How to run this file:
// 1. [obtain firefox source code]
// 2. [build/obtain firefox binaries]
// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
// [path to]/genHPKPStaticpins.js \
// [absolute path to]/PreloadedHPKPins.json \
// [an unused argument - see bug 1205406] \
-// [absolute path to]/StaticHPKPins.h
+// [absolute path to]/StaticHPKPins.h \
+// [absolute path to]/StaticHPKPData.json \
+// [absolute path to]/version.txt
"use strict";
-if (arguments.length != 3) {
+if (arguments.length < 3) {
throw new Error("Usage: genHPKPStaticPins.js " +
"<absolute path to PreloadedHPKPins.json> " +
"<an unused argument - see bug 1205406> " +
- "<absolute path to StaticHPKPins.h>");
+ "<absolute path to StaticHPKPins.h>" +
+ "(optional) <absolute path to StaticHPKPData.json>" +
+ "(optional) <absolute path to version.txt");
}
const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
var { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
var { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
var { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
@@ -68,16 +72,37 @@ var gStaticPins = parseJson(arguments[0]
// arguments[1] is ignored for now. See bug 1205406.
// Open the output file.
var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
file.initWithPath(arguments[2]);
var gFileOutputStream = FileUtils.openSafeFileOutputStream(file);
+let gJsonFileOutputStream;
+if (arguments.length > 3) {
+ var jsonFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+ jsonFile.initWithPath(arguments[3]);
+ gJsonFileOutputStream = FileUtils.openSafeFileOutputStream(jsonFile);
+}
+
+let gVersion = "unknown";
+if (arguments.length > 4) {
+ gVersion = readFileToString(arguments[4]).trim();
+}
+
+let gJSONReportData = {"fingerprints": [],
+ "pinsets": [],
+ "domainlists": [],
+ "creationInfo": {
+ "version": gVersion,
+ "date": (new Date()).toISOString()
+ }};
+
+
function writeString(string) {
gFileOutputStream.write(string, string.length);
}
function readFileToString(filename) {
let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
file.initWithPath(filename);
let stream = Cc["@mozilla.org/network/file-input-stream;1"]
@@ -522,16 +547,17 @@ function writeDomainList(chromeImportedE
chromeImportedEntries.splice(i, 1);
}
}
let sortedEntries = gStaticPins.entries;
sortedEntries.push.apply(sortedEntries, chromeImportedEntries);
for (let entry of sortedEntries.sort(compareByName)) {
count++;
writeEntry(entry);
+ gJSONReportData.domainlists.push(entry);
}
writeString("};\n");
writeString("\n// Pinning Preload List Length = " + count + ";\n");
writeString("\nstatic const int32_t kUnknownId = -1;\n");
}
function writeFile(certNameToSKD, certSKDToName,
@@ -557,40 +583,48 @@ function writeFile(certNameToSKD, certSK
// Write actual fingerprints.
Object.keys(usedFingerprints).sort().forEach(function(certName) {
if (certName) {
writeString("/* " + certName + " */\n");
writeString("static const char " + nameToAlias(certName) + "[] =\n");
writeString(" \"" + certNameToSKD[certName] + "\";\n");
writeString("\n");
+ gJSONReportData.fingerprints.push({"name": certName, "skd": certNameToSKD[certName]});
}
});
// Write the pinsets
writeString(PINSETDEF);
writeString("/* PreloadedHPKPins.json pinsets */\n");
gStaticPins.pinsets.sort(compareByName).forEach(function(pinset) {
writeFullPinset(certNameToSKD, certSKDToName, pinset);
+ gJSONReportData.pinsets.push({"source": "PreloadedHPKPins.json pinset", "name": pinset.name, "hashes": pinset.sha256_hashes});
});
writeString("/* Chrome static pinsets */\n");
for (let key in chromeImportedPinsets) {
if (mozillaPins[key]) {
dump("Skipping duplicate pinset " + key + "\n");
} else {
dump("Writing pinset " + key + "\n");
writeFullPinset(certNameToSKD, certSKDToName, chromeImportedPinsets[key]);
+ gJSONReportData.pinsets.push({"source": "Chrome static pinset", "name": chromeImportedPinsets[key].name, "hashes": chromeImportedPinsets[key].sha256_hashes});
}
}
// Write the domainlist entries.
writeString(DOMAINHEADER);
writeDomainList(chromeImportedEntries);
writeString("\n");
writeString(genExpirationTime());
+
+ if (gJsonFileOutputStream) {
+ let reportString = JSON.stringify(gJSONReportData);
+ gJsonFileOutputStream.write(reportString, reportString.length);
+ }
}
function loadExtraCertificates(certStringList) {
let constructedCerts = [];
for (let certString of certStringList) {
constructedCerts.push(gCertDB.constructX509FromBase64(certString));
}
return constructedCerts;
@@ -603,8 +637,11 @@ var [ chromeNameToHash, chromeNameToMozN
var [ chromeImportedPinsets, chromeImportedEntries ] =
downloadAndParseChromePins(gStaticPins.chromium_data.json_file_url,
chromeNameToHash, chromeNameToMozName, certNameToSKD, certSKDToName);
writeFile(certNameToSKD, certSKDToName, chromeImportedPinsets,
chromeImportedEntries);
FileUtils.closeSafeFileOutputStream(gFileOutputStream);
+if (gJsonFileOutputStream) {
+ FileUtils.closeSafeFileOutputStream(gJsonFileOutputStream);
+}
--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -3,19 +3,21 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
// How to run this file:
// 1. [obtain firefox source code]
// 2. [build/obtain firefox binaries]
// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
// [path to]/getHSTSPreloadlist.js \
-// [absolute path to]/nsSTSPreloadlist.inc'
+// [absolute path to]/nsSTSPreloadlist.inc' \
+// [absolute path to]/StaticHSTSData.json
// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
-// working directory.
+// working directory and creates a JSON file at the path specified for
+// StaticHSTSData.json
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cu = Components.utils;
var Cr = Components.results;
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/FileUtils.jsm");
@@ -38,16 +40,18 @@ const HEADER = "/* This Source Code Form
"\n" +
"/*****************************************************************************/\n" +
"/* This is an automatically generated file. If you're not */\n" +
"/* nsSiteSecurityService.cpp, you shouldn't be #including it. */\n" +
"/*****************************************************************************/\n" +
"\n" +
"#include <stdint.h>\n";
+var gFilename;
+
function download() {
var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance(Ci.nsIXMLHttpRequest);
req.open("GET", SOURCE, false); // doing the request synchronously
try {
req.send();
} catch (e) {
throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
@@ -220,45 +224,65 @@ function compareHSTSStatus(a, b) {
function writeTo(string, fos) {
fos.write(string, string.length);
}
// Determines and returns a string representing a declaration of when this
// preload list should no longer be used.
// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
-function getExpirationTimeString() {
+function getExpirationTime() {
var now = new Date();
var nowMillis = now.getTime();
// MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
- var expirationMicros = expirationMillis * 1000;
+ return expirationMillis * 1000;
+}
+
+function getExpirationTimeString(expirationMicros) {
return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
}
function errorToString(status) {
return (status.error == ERROR_MAX_AGE_TOO_LOW
? status.error + status.maxAge
: status.error);
}
function writeEntry(status, indices, outputStream) {
let includeSubdomains = (status.finalIncludeSubdomains ? "true" : "false");
writeTo(" { " + indices[status.name] + ", " + includeSubdomains + " },\n",
outputStream);
}
function output(sortedStatuses, currentList) {
+
try {
var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
var fos = FileUtils.openSafeFileOutputStream(file);
var eos = FileUtils.openSafeFileOutputStream(errorFile);
writeTo(HEADER, fos);
- writeTo(getExpirationTimeString(), fos);
+ var expirationTime = getExpirationTime();
+ writeTo(getExpirationTimeString(expirationTime), fos);
+
+ let reportData = {
+ "statuses":[],
+ "creationInfo": {
+ "date": (new Date()).toISOString(),
+ "expiration": expirationTime
+ }
+ };
+
+ let jsonFileOutputStream;
+ if (gFilename) {
+ var jsonFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+ jsonFile.initWithPath(gFilename);
+ jsonFileOutputStream = FileUtils.openSafeFileOutputStream(jsonFile);
+ }
for (let status in sortedStatuses) {
// If we've encountered an error for this entry (other than the site not
// sending an HSTS header), be safe and don't remove it from the list
// (given that it was already on the list).
if (status.error != ERROR_NONE &&
status.error != ERROR_NO_HSTS_HEADER &&
status.error != ERROR_MAX_AGE_TOO_LOW &&
@@ -306,16 +330,17 @@ function output(sortedStatuses, currentL
// Add 1 for the null terminator in C.
currentIndex += status.name.length + 1;
// Rebuilding the preload list requires reading the previous preload
// list. Write out a comment describing each host prior to writing out
// the string for the host.
writeTo(" /* \"" + status.name + "\", " +
(status.finalIncludeSubdomains ? "true" : "false") + " */ ",
fos);
+ reportData.statuses.push({"name": status.name, "includeSubdomains": status.finalIncludeSubdomains});
// Write out the string itself as individual characters, including the
// null terminator. We do it this way rather than using C's string
// concatentation because some compilers have hardcoded limits on the
// lengths of string literals, and the preload list is large enough
// that it runs into said limits.
for (let c of status.name) {
writeTo("'" + c + "', ", fos);
}
@@ -336,16 +361,21 @@ function output(sortedStatuses, currentL
writeTo(PREFIX, fos);
for (let status of includedStatuses) {
writeEntry(status, indices, fos);
}
writeTo(POSTFIX, fos);
FileUtils.closeSafeFileOutputStream(fos);
FileUtils.closeSafeFileOutputStream(eos);
+
+ if (jsonFileOutputStream) {
+ writeTo(JSON.stringify(reportData), jsonFileOutputStream);
+ FileUtils.closeSafeFileOutputStream(jsonFileOutputStream);
+ }
} catch (e) {
dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
}
}
function shouldRetry(response) {
return (response.error != ERROR_NO_HSTS_HEADER &&
response.error != ERROR_MAX_AGE_TOO_LOW &&
@@ -455,22 +485,27 @@ function insertTestHosts(hstsStatuses) {
forceInclude: true,
originalIncludeSubdomains: testEntry.includeSubdomains,
});
}
}
// ****************************************************************************
// This is where the action happens:
-if (arguments.length != 1) {
+if (arguments.length < 1) {
throw new Error("Usage: getHSTSPreloadList.js " +
- "<absolute path to current nsSTSPreloadList.inc>");
+ "<absolute path to current nsSTSPreloadList.inc> " +
+ "<absolute path to StaticHSTSData.json>");
}
// get the current preload list
var currentHosts = readCurrentList(arguments[0]);
+// get the filename for the JSON report
+if (arguments.length > 1) {
+ gFilename = arguments[1];
+}
// delete any hosts we use in tests so we don't actually connect to them
deleteTestHosts(currentHosts);
// disable the current preload list so it won't interfere with requests we make
Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
// download and parse the raw json file from the Chromium source
var rawdata = download();
// get just the hosts with mode: "force-https"
var hosts = getHosts(rawdata);