--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -88,17 +88,18 @@ function getHosts(rawdata) {
for (let entry of rawdata.entries) {
if (entry.mode && entry.mode == "force-https") {
if (entry.name) {
// We trim the entry name here to avoid malformed URI exceptions when we
// later try to connect to the domain.
entry.name = entry.name.trim();
entry.retries = MAX_RETRIES;
- entry.originalIncludeSubdomains = entry.include_subdomains;
+ // We prefer the camelCase variable to the JSON's snake case version
+ entry.includeSubdomains = entry.include_subdomains;
hosts.push(entry);
} else {
throw new Error("ERROR: entry not formatted correctly: no name found");
}
}
}
return hosts;
@@ -126,29 +127,26 @@ function processStsHeader(host, header,
error = e;
}
} else if (status == 0) {
error = ERROR_CONNECTING_TO_HOST;
} else {
error = ERROR_NO_HSTS_HEADER;
}
- let forceInclude = (host.forceInclude || host.pins == "google");
-
if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
error = ERROR_MAX_AGE_TOO_LOW;
}
return { name: host.name,
maxAge: maxAge.value,
includeSubdomains: includeSubdomains.value,
error,
retries: host.retries - 1,
- forceInclude,
- originalIncludeSubdomains: host.originalIncludeSubdomains };
+ forceInclude: host.forceInclude };
}
// RedirectAndAuthStopper prevents redirects and HTTP authentication
function RedirectAndAuthStopper() {}
RedirectAndAuthStopper.prototype = {
// nsIChannelEventSink
asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) {
@@ -173,16 +171,18 @@ RedirectAndAuthStopper.prototype = {
};
function getHSTSStatus(host, resultList) {
var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance(Ci.nsIXMLHttpRequest);
var inResultList = false;
var uri = "https://" + host.name + "/";
req.open("GET", uri, true);
+ req.setRequestHeader("X-Automated-Tool",
+ "https://hg.mozilla.org/mozilla-central/file/tip/security/manager/tools/getHSTSPreloadList.js");
req.timeout = REQUEST_TIMEOUT;
let errorhandler = (evt) => {
dump(`ERROR: error making request to ${host.name} (type=${evt.type})\n`);
if (!inResultList) {
inResultList = true;
resultList.push(processStsHeader(host, null, req.status,
req.channel.securityInfo));
@@ -249,17 +249,18 @@ function output(sortedStatuses, currentL
var eos = FileUtils.openSafeFileOutputStream(errorFile);
writeTo(HEADER, fos);
writeTo(getExpirationTimeString(), fos);
for (let status of sortedStatuses) {
// If we've encountered an error for this entry (other than the site not
// sending an HSTS header), be safe and don't remove it from the list
// (given that it was already on the list).
- if (status.error != ERROR_NONE &&
+ if (!status.forceInclude &&
+ status.error != ERROR_NONE &&
status.error != ERROR_NO_HSTS_HEADER &&
status.error != ERROR_MAX_AGE_TOO_LOW &&
status.name in currentList) {
dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
writeTo(status.name + ": " + errorToString(status) + "\n", eos);
status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
status.includeSubdomains = currentList[status.name];
}
@@ -268,40 +269,29 @@ function output(sortedStatuses, currentL
// Filter out entries we aren't including.
var includedStatuses = sortedStatuses.filter(function (status) {
if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
dump("INFO: " + status.name + " NOT ON the preload list\n");
writeTo(status.name + ": " + errorToString(status) + "\n", eos);
return false;
}
- dump("INFO: " + status.name + " ON the preload list\n");
+ dump("INFO: " + status.name + " ON the preload list (includeSubdomains: "
+ + status.includeSubdomains + "\n");
if (status.forceInclude && status.error != ERROR_NONE) {
writeTo(status.name + ": " + errorToString(status) + " (error "
+ "ignored - included regardless)\n", eos);
}
return true;
});
- // Resolve whether we should include subdomains for each entry. We could
- // do this while writing out entries, but separating out that decision is
- // clearer. Making that decision here also means we can write the choices
- // in the comments in the static string table, which makes parsing the
- // current list significantly easier when we go to update the list.
- for (let status of includedStatuses) {
- let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
- ? status.originalIncludeSubdomains
- : status.includeSubdomains);
- status.finalIncludeSubdomains = incSubdomainsBool;
- }
-
writeTo(GPERF_DELIM, fos);
for (let status of includedStatuses) {
- let includeSubdomains = (status.finalIncludeSubdomains ? 1 : 0);
+ let includeSubdomains = (status.includeSubdomains ? 1 : 0);
writeTo(status.name + ", " + includeSubdomains + "\n", fos);
}
writeTo(GPERF_DELIM, fos);
FileUtils.closeSafeFileOutputStream(fos);
FileUtils.closeSafeFileOutputStream(eos);
} catch (e) {
dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
@@ -309,43 +299,46 @@ function output(sortedStatuses, currentL
}
function shouldRetry(response) {
return (response.error != ERROR_NO_HSTS_HEADER &&
response.error != ERROR_MAX_AGE_TOO_LOW &&
response.error != ERROR_NONE && response.retries > 0);
}
-function getHSTSStatuses(inHosts, outStatuses) {
- var expectedOutputLength = inHosts.length;
- var tmpOutput = [];
- for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
+function probeHSTSStatuses(inHosts) {
+ let outStatuses = [];
+ let expectedOutputLength = inHosts.length;
+ let tmpOutput = [];
+ for (let i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
let host = inHosts.shift();
dump("spinning off request to '" + host.name + "' (remaining retries: " +
host.retries + ")\n");
getHSTSStatus(host, tmpOutput);
}
while (outStatuses.length != expectedOutputLength) {
waitForAResponse(tmpOutput);
- var response = tmpOutput.shift();
+ let response = tmpOutput.shift();
dump("request to '" + response.name + "' finished\n");
if (shouldRetry(response)) {
inHosts.push(response);
} else {
outStatuses.push(response);
}
if (inHosts.length > 0) {
let host = inHosts.shift();
dump("spinning off request to '" + host.name + "' (remaining retries: " +
host.retries + ")\n");
getHSTSStatus(host, tmpOutput);
}
}
+
+ return outStatuses;
}
// Since all events are processed on the main thread, and since event
// handlers are not preemptible, there shouldn't be any concurrency issues.
function waitForAResponse(outputList) {
// From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
Services.tm.spinEventLoopUntil(() => outputList.length != 0);
}
@@ -401,53 +394,84 @@ const TEST_ENTRIES = [
];
function deleteTestHosts(currentHosts) {
for (let testEntry of TEST_ENTRIES) {
delete currentHosts[testEntry.name];
}
}
-function insertTestHosts(hstsStatuses) {
+function getTestHosts() {
+ let hosts = [];
for (let testEntry of TEST_ENTRIES) {
- hstsStatuses.push({
+ hosts.push({
name: testEntry.name,
maxAge: MINIMUM_REQUIRED_MAX_AGE,
includeSubdomains: testEntry.includeSubdomains,
error: ERROR_NONE,
// This deliberately doesn't have a value for `retries` (because we should
// never attempt to connect to this host).
forceInclude: true,
- originalIncludeSubdomains: testEntry.includeSubdomains,
});
}
+ return hosts;
+}
+
+function insertHosts(inoutHostList, inAddedHosts) {
+ for (let host of inAddedHosts) {
+ inoutHostList.push(host);
+ }
+}
+
+function filterForcedInclusions(inHosts, outNotForced, outForced) {
+ // Apply our filters (based on policy today) to determine which entries
+ // will be included without being checked (forced); the others will be
+ // checked using active probing.
+ for (let host of inHosts) {
+ if (host.policy == "google" || host.policy == "public-suffix" ||
+ host.policy == "public-suffix-requested") {
+ host.forceInclude = true;
+ host.error = ERROR_NONE;
+ outForced.push(host);
+ } else {
+ outNotForced.push(host);
+ }
+ }
}
// ****************************************************************************
// This is where the action happens:
if (arguments.length != 1) {
throw new Error("Usage: getHSTSPreloadList.js " +
"<absolute path to current nsSTSPreloadList.inc>");
}
// get the current preload list
-var currentHosts = readCurrentList(arguments[0]);
+let currentHosts = readCurrentList(arguments[0]);
// delete any hosts we use in tests so we don't actually connect to them
deleteTestHosts(currentHosts);
// disable the current preload list so it won't interfere with requests we make
Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
// download and parse the raw json file from the Chromium source
-var rawdata = download();
+let rawdata = download();
// get just the hosts with mode: "force-https"
-var hosts = getHosts(rawdata);
+let hosts = getHosts(rawdata);
// add hosts in the current list to the new list (avoiding duplicates)
combineLists(hosts, currentHosts);
-// get the HSTS status of each host
-var hstsStatuses = [];
-getHSTSStatuses(hosts, hstsStatuses);
-// add the hosts we use in tests
-insertTestHosts(hstsStatuses);
-// sort the hosts alphabetically
+// Don't contact hosts that are forced to be included anyway
+let hostsToContact = [];
+let forcedHosts = [];
+filterForcedInclusions(hosts, hostsToContact, forcedHosts);
+// Initialize the final status list
+let hstsStatuses = [];
+// Add the hosts we use in tests
+insertHosts(hstsStatuses, getTestHosts());
+// Add in the hosts that are forced
+insertHosts(hstsStatuses, forcedHosts);
+// Probe the HSTS status of each host and add them into hstsStatuses
+let probedStatuses = probeHSTSStatuses(hostsToContact);
+insertHosts(hstsStatuses, probedStatuses);
+// Sort the hosts alphabetically
hstsStatuses.sort(compareHSTSStatus);
-// write the results to a file (this is where we filter out hosts that we
+// Write the results to a file (this is where we filter out hosts that we
// either couldn't connect to, didn't receive an HSTS header from, couldn't
// parse the header, or had a header with too short a max-age)
output(hstsStatuses, currentHosts);
// ****************************************************************************