Bug 1391703 - Introduce common JSM for security/manager/tools/ scripts. r=keeler draft
authorCykesiopka <cykesiopka.bmo@gmail.com>
Wed, 23 Aug 2017 14:46:37 +0800
changeset 651115 5cfdf7fb72f3c0880bcaaf2060c7fcc6645f296a
parent 650941 7c50f0c999c5bf8ee915261997597a5a9b8fb2ae
child 727586 42f3de473023f9153451d20af8593e93b01a0681
push id75594
push usercykesiopka.bmo@gmail.com
push dateWed, 23 Aug 2017 07:36:33 +0000
reviewerskeeler
bugs1391703
milestone57.0a1
Bug 1391703 - Introduce common JSM for security/manager/tools/ scripts. r=keeler PSM has various xpcshell scripts under the security/manager/tools/ folder. At the moment, these scripts: 1. Duplicate code. 2. Aren't testable. This patch introduces a common, unit tested JSM that these scripts can use. MozReview-Commit-ID: 5NKRUeJgG8f
security/manager/moz.build
security/manager/tools/.eslintrc.js
security/manager/tools/PSMToolUtils.jsm
security/manager/tools/dumpGoogleRoots.js
security/manager/tools/genHPKPStaticPins.js
security/manager/tools/genRootCAHashes.js
security/manager/tools/getHSTSPreloadList.js
security/manager/tools/moz.build
security/manager/tools/tests/.eslintrc.js
security/manager/tools/tests/test_psmtoolutils.js
security/manager/tools/tests/xpcshell.ini
--- a/security/manager/moz.build
+++ b/security/manager/moz.build
@@ -2,12 +2,16 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files("**"):
     BUG_COMPONENT = ("Core", "Security: PSM")
 
-DIRS += ['ssl', 'locales']
+DIRS += [
+    'locales',
+    'ssl',
+    'tools',
+]
 
 if CONFIG['MOZ_XUL'] and CONFIG['MOZ_BUILD_APP'] != 'mobile/android':
     DIRS += ['pki']
--- a/security/manager/tools/.eslintrc.js
+++ b/security/manager/tools/.eslintrc.js
@@ -1,9 +1,10 @@
 "use strict";
 
 module.exports = {
   "globals": {
     // JS files in this folder are commonly xpcshell scripts where |arguments|
-    // is defined in the global scope.
+    // and |__LOCATION__| are defined in the global scope.
+    "__LOCATION__": false,
     "arguments": false
   }
 };
new file mode 100644
--- /dev/null
+++ b/security/manager/tools/PSMToolUtils.jsm
@@ -0,0 +1,59 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Common code for PSM scripts in security/manager/tools/.
+
+"use strict";
+
+const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+
+/**
+ * Synchronously downloads a file.
+ *
+ * @param {String} url
+ *        The URL to download the file from.
+ * @param {Boolean} decodeContentsAsBase64
+ *        Whether the downloaded contents should be Base64 decoded before being
+ *        returned.
+ * @returns {String}
+ *          The downloaded contents.
+ */
+function downloadFile(url, decodeContentsAsBase64) {
+  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+              .createInstance(Ci.nsIXMLHttpRequest);
+  req.open("GET", url, false); // doing the request synchronously
+  try {
+    req.send();
+  } catch (e) {
+    throw new Error(`ERROR: problem downloading '${url}': ${e}`);
+  }
+
+  if (req.status != 200) {
+    throw new Error(`ERROR: problem downloading '${url}': status ${req.status}`);
+  }
+
+  if (!decodeContentsAsBase64) {
+    return req.responseText;
+  }
+
+  try {
+    return atob(req.responseText);
+  } catch (e) {
+    throw new Error(`ERROR: could not decode data as base64 from '${url}': ` +
+                    e);
+  }
+}
+
+/**
+ * Removes //-style block (but not trailing) comments from a string.
+ * @param {String} input
+ *        Potentially multi-line input.
+ * @returns {String}
+ */
+function stripComments(input) {
+  return input.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
+}
+
+this.PSMToolUtils = { downloadFile, stripComments };
+this.EXPORTED_SYMBOLS = ["PSMToolUtils"];
--- a/security/manager/tools/dumpGoogleRoots.js
+++ b/security/manager/tools/dumpGoogleRoots.js
@@ -8,35 +8,23 @@
 //
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell dumpGoogleRoots.js'
 // 4. [paste the output into the appropriate section in
 //     security/manager/tools/PreloadedHPKPins.json]
 
-var Cc = Components.classes;
-var Ci = Components.interfaces;
+const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+
+const { PSMToolUtils } =
+  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
 
 function downloadRoots() {
-  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-              .createInstance(Ci.nsIXMLHttpRequest);
-  req.open("GET", "https://pki.google.com/roots.pem", false);
-  try {
-    req.send();
-  } catch (e) {
-    throw new Error("ERROR: problem downloading Google Root PEMs: " + e);
-  }
-
-  if (req.status != 200) {
-    throw new Error("ERROR: problem downloading Google Root PEMs. Status: " +
-                    req.status);
-  }
-
-  let pem = req.responseText;
+  let pem = PSMToolUtils.downloadFile("https://pki.google.com/roots.pem", false);
   let roots = [];
   let currentPEM = "";
   let readingRoot = false;
   let certDB = Cc["@mozilla.org/security/x509certdb;1"]
                  .getService(Ci.nsIX509CertDB);
   for (let line of pem.split(/[\r\n]/)) {
     if (line == "-----END CERTIFICATE-----") {
       if (currentPEM) {
--- a/security/manager/tools/genHPKPStaticPins.js
+++ b/security/manager/tools/genHPKPStaticPins.js
@@ -16,19 +16,21 @@ if (arguments.length != 3) {
   throw new Error("Usage: genHPKPStaticPins.js " +
                   "<absolute path to PreloadedHPKPins.json> " +
                   "<an unused argument - see bug 1205406> " +
                   "<absolute path to StaticHPKPins.h>");
 }
 
 const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
 
-var { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
-var { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
-var { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
+const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
+const { PSMToolUtils } =
+  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 
 var gCertDB = Cc["@mozilla.org/security/x509certdb;1"]
                 .getService(Ci.nsIX509CertDB);
 
 const SHA256_PREFIX = "sha256/";
 const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_";
 
 // Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable)
@@ -82,57 +84,19 @@ function readFileToString(filename) {
   file.initWithPath(filename);
   let stream = Cc["@mozilla.org/network/file-input-stream;1"]
                  .createInstance(Ci.nsIFileInputStream);
   stream.init(file, -1, 0, 0);
   let buf = NetUtil.readInputStreamToString(stream, stream.available());
   return buf;
 }
 
-function stripComments(buf) {
-  let lines = buf.split("\n");
-  let entryRegex = /^\s*\/\//;
-  let data = "";
-  for (let i = 0; i < lines.length; ++i) {
-    let match = entryRegex.exec(lines[i]);
-    if (!match) {
-      data = data + lines[i];
-    }
-  }
-  return data;
-}
-
-function download(filename) {
-  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-              .createInstance(Ci.nsIXMLHttpRequest);
-  req.open("GET", filename, false); // doing the request synchronously
-  try {
-    req.send();
-  } catch (e) {
-    throw new Error(`ERROR: problem downloading '${filename}': ${e}`);
-  }
-
-  if (req.status != 200) {
-    throw new Error("ERROR: problem downloading '" + filename + "': status " +
-                    req.status);
-  }
-
-  let resultDecoded;
-  try {
-    resultDecoded = atob(req.responseText);
-  } catch (e) {
-    throw new Error("ERROR: could not decode data as base64 from '" + filename +
-                    "': " + e);
-  }
-  return resultDecoded;
-}
-
 function downloadAsJson(filename) {
-  // we have to filter out '//' comments, while not mangling the json
-  let result = download(filename).replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
+  let jsonWithComments = PSMToolUtils.downloadFile(filename, true);
+  let result = PSMToolUtils.stripComments(jsonWithComments);
   let data = null;
   try {
     data = JSON.parse(result);
   } catch (e) {
     throw new Error("ERROR: could not parse data from '" + filename + "': " + e);
   }
   return data;
 }
@@ -204,17 +168,17 @@ function downloadAndParseChromeCerts(fil
 
   // Parsing states.
   const PRE_NAME = 0;
   const POST_NAME = 1;
   const IN_CERT = 2;
   const IN_PUB_KEY = 3;
   let state = PRE_NAME;
 
-  let lines = download(filename).split("\n");
+  let lines = PSMToolUtils.downloadFile(filename, true).split("\n");
   let pemCert = "";
   let pemPubKey = "";
   let hash = "";
   let chromeNameToHash = {};
   let chromeNameToMozName = {};
   let chromeName;
   for (let line of lines) {
     // Skip comments and newlines.
@@ -403,17 +367,17 @@ function loadNSSCertinfo(extraCertificat
     let SKD = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=";
     certNameToSKD[name] = SKD;
     certSKDToName[SKD] = name;
   }
   return [certNameToSKD, certSKDToName];
 }
 
 function parseJson(filename) {
-  let json = stripComments(readFileToString(filename));
+  let json = PSMToolUtils.stripComments(readFileToString(filename));
   return JSON.parse(json);
 }
 
 function nameToAlias(certName) {
   // change the name to a string valid as a c identifier
   // remove  non-ascii characters
   certName = certName.replace(/[^[:ascii:]]/g, "_");
   // replace non word characters
--- a/security/manager/tools/genRootCAHashes.js
+++ b/security/manager/tools/genRootCAHashes.js
@@ -4,28 +4,27 @@
 "use strict";
 
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell genRootCAHashes.js \
 //                                  [absolute path to]/RootHashes.inc'
 
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
-var Cr = Components.results;
+const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
 
 const nsX509CertDB = "@mozilla.org/security/x509certdb;1";
 const CertDb = Components.classes[nsX509CertDB].getService(Ci.nsIX509CertDB);
 
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/FileUtils.jsm");
-Cu.import("resource://gre/modules/NetUtil.jsm");
 const { CommonUtils } = Cu.import("resource://services-common/utils.js", {});
+const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
+const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
+const { PSMToolUtils } =
+  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 
 const FILENAME_OUTPUT = "RootHashes.inc";
 const FILENAME_TRUST_ANCHORS = "KnownRootHashes.json";
 const ROOT_NOT_ASSIGNED = -1;
 
 const JSON_HEADER = "// This Source Code Form is subject to the terms of the Mozilla Public\n" +
 "// License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
 "// file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
@@ -74,38 +73,24 @@ function hexSlice(bytes, start, end) {
     ret += "0x" + hex;
     if (i < end - 1) {
       ret += ", ";
     }
   }
   return ret;
 }
 
-function stripComments(buf) {
-  let lines = buf.split("\n");
-  let entryRegex = /^\s*\/\//;
-  let data = "";
-  for (let i = 0; i < lines.length; i++) {
-    let match = entryRegex.exec(lines[i]);
-    if (!match) {
-      data = data + lines[i];
-    }
-  }
-  return data;
-}
-
-
 // Load the trust anchors JSON object from disk
 function loadTrustAnchors(file) {
   if (file.exists()) {
     let stream = Cc["@mozilla.org/network/file-input-stream;1"]
                    .createInstance(Ci.nsIFileInputStream);
     stream.init(file, -1, 0, 0);
     let buf = NetUtil.readInputStreamToString(stream, stream.available());
-    return JSON.parse(stripComments(buf));
+    return JSON.parse(PSMToolUtils.stripComments(buf));
   }
   // If there's no input file, bootstrap.
   return { roots: [], maxBin: 0 };
 }
 
 // Saves our persistence file so that we don't lose track of the mapping
 // between bin numbers and the CA-hashes, even as CAs come and go.
 function writeTrustAnchors(file) {
--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -7,24 +7,23 @@
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/getHSTSPreloadlist.js \
 //                                  [absolute path to]/nsSTSPreloadlist.inc'
 // Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
 //       working directory.
 
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
-var Cr = Components.results;
+const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
 
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/FileUtils.jsm");
-Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
+const { PSMToolUtils } =
+  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+const { XPCOMUtils } = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
 
 const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT";
 const OUTPUT = "nsSTSPreloadList.inc";
 const ERROR_OUTPUT = "nsSTSPreloadList.errors";
 const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
 const MAX_CONCURRENT_REQUESTS = 5;
 const MAX_RETRIES = 3;
 const REQUEST_TIMEOUT = 30 * 1000;
@@ -40,40 +39,18 @@ const HEADER = "/* This Source Code Form
 "/* This is an automatically generated file. If you're not                    */\n" +
 "/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
 const GPERF_DELIM = "%%\n";
 
 function download() {
-  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-            .createInstance(Ci.nsIXMLHttpRequest);
-  req.open("GET", SOURCE, false); // doing the request synchronously
-  try {
-    req.send();
-  } catch (e) {
-    throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
-  }
-
-  if (req.status != 200) {
-    throw new Error("ERROR: problem downloading '" + SOURCE + "': status " +
-                    req.status);
-  }
-
-  var resultDecoded;
-  try {
-    resultDecoded = atob(req.responseText);
-  } catch (e) {
-    throw new Error("ERROR: could not decode data as base64 from '" + SOURCE +
-                    "': " + e);
-  }
-
-  // we have to filter out '//' comments, while not mangling the json
-  var result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
+  let resultDecoded = PSMToolUtils.downloadFile(SOURCE, true);
+  let result = PSMToolUtils.stripComments(resultDecoded);
   var data = null;
   try {
     data = JSON.parse(result);
   } catch (e) {
     throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
   }
   return data;
 }
new file mode 100644
--- /dev/null
+++ b/security/manager/tools/moz.build
@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini']
+
+# This JSM is only used by the xpcshell scripts in this directory and not by
+# Firefox itself, so we only register the JSM as a test module to:
+#  - Avoid bloat.
+#  - Make tests slightly cleaner.
+# xpcshell scripts can instead import the JSM using a file:// URL constructed
+# using the built in |__LOCATION__| nsIFile constant.
+TESTING_JS_MODULES.psm += ['PSMToolUtils.jsm']
new file mode 100644
--- /dev/null
+++ b/security/manager/tools/tests/.eslintrc.js
@@ -0,0 +1,5 @@
+"use strict";
+
+module.exports = {
+  "extends": "plugin:mozilla/xpcshell-test"
+};
new file mode 100644
--- /dev/null
+++ b/security/manager/tools/tests/test_psmtoolutils.js
@@ -0,0 +1,108 @@
+// -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+// Any copyright is dedicated to the Public Domain.
+// http://creativecommons.org/publicdomain/zero/1.0/
+"use strict";
+
+// Tests PSMToolUtils.jsm.
+
+const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+
+const { HttpServer } = Cu.import("resource://testing-common/httpd.js", {});
+const { PSMToolUtils } =
+  Cu.import("resource://testing-common/psm/PSMToolUtils.jsm", {});
+const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+
+const ResponseType = {
+  BadStatusCode: { contents: "", statusCode: 404 },
+  Base64: { contents: "eyJmb28iOiJiYXIifQ==" }, // btoa('{"foo":"bar"}')
+  NotBase64: { contents: "not*base 64." },
+};
+
+var gHttpServer = new HttpServer();
+var gHttpServerPrePath;
+
+add_task(function testDownloadFile_Setup() {
+  gHttpServer = new HttpServer();
+  gHttpServer.registerPrefixHandler("/", function(request, response) {
+    let pathWithoutLeadingSlash = request.path.substring(1);
+    let responseType = ResponseType[pathWithoutLeadingSlash];
+    ok(responseType,
+       `'${pathWithoutLeadingSlash}' should be a valid ResponseType`);
+
+    response.setStatusLine(request.httpVersion, responseType.statusCode || 200,
+                           null);
+    response.setHeader("Content-Type", "text/plain");
+    response.write(responseType.contents);
+  });
+  gHttpServer.start(-1);
+  do_register_cleanup(() => {
+    gHttpServer.stop(() => {});
+  });
+  gHttpServerPrePath = `http://localhost:${gHttpServer.identity.primaryPort}`;
+});
+
+add_task(function testDownloadFile_BadStatusCode() {
+  let url = `${gHttpServerPrePath}/BadStatusCode`;
+  throws(() => PSMToolUtils.downloadFile(url, true),
+         /problem downloading.*status 404/,
+         "Exception should be thrown for a non-200 status code");
+});
+
+add_task(function testDownloadFile_DecodeAsBase64_Base64() {
+  let url = `${gHttpServerPrePath}/Base64`;
+  let contents = PSMToolUtils.downloadFile(url, true);
+  equal(contents, '{"foo":"bar"}', "Should get back decoded Base64 contents");
+});
+
+add_task(function testDownloadFile_DecodeAsBase64_NotBase64() {
+  let url = `${gHttpServerPrePath}/NotBase64`;
+  throws(() => PSMToolUtils.downloadFile(url, true),
+         /could not decode data as base64/,
+         "Exception should be thrown if we get non-Base64 content when we " +
+         "were expecting otherwise");
+});
+
+add_task(function testDownloadFile_DontDecodeAsBase64_Base64() {
+  let url = `${gHttpServerPrePath}/Base64`;
+  let contents = PSMToolUtils.downloadFile(url, false);
+  equal(contents, ResponseType.Base64.contents,
+        "Should get back undecoded Base64 contents");
+});
+
+add_task(function testDownloadFile_DontDecodeAsBase64_NotBase64() {
+  let url = `${gHttpServerPrePath}/NotBase64`;
+  let contents = PSMToolUtils.downloadFile(url, false);
+  equal(contents, ResponseType.NotBase64.contents,
+        "Should get back expected contents");
+});
+
+add_task(function testDownloadFile_InvalidDomain() {
+  let domain = "test-psmtoolutils.js.nosuchdomain.test";
+  Services.prefs.setCharPref("network.dns.localDomains", domain);
+  throws(() => PSMToolUtils.downloadFile(`http://${domain}/foo`, true),
+         /problem downloading/,
+         "Exception should be thrown if trying to download from an invalid " +
+         "domain");
+});
+
+add_task(function testStripComments() {
+  const tests = [
+    {input: "", expected: ""},
+    {input: "{}", expected: "{}"},
+    {input: "     //\n", expected: ""},
+    {input: "\t//\n//\n   //\n", expected: ""},
+    {input: '{"foo":"bar"}', expected: '{"foo":"bar"}'},
+    // From Bug 1197607.
+    {input: '"report_uri": "http://clients3.google.com/cert_upload_json"',
+     expected: '"report_uri": "http://clients3.google.com/cert_upload_json"'},
+    {input: '//abc\n{"foo":"bar"}', expected: '{"foo":"bar"}'},
+    {input: '{"foo":"bar",\n//baz\n"aaa": "bbb"}',
+     expected: '{"foo":"bar",\n"aaa": "bbb"}'},
+    {input: "// foo http://example.com/baz\n", expected: ""},
+  ];
+  for (let test of tests) {
+    let result = PSMToolUtils.stripComments(test.input);
+    equal(test.expected, result,
+          `Expected and actual result should match for input '${test.input}'`);
+  }
+});
new file mode 100644
--- /dev/null
+++ b/security/manager/tools/tests/xpcshell.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+tags = psm
+
+[test_psmtoolutils.js]