Bug 1309351: Part 2 - Speed up synchronous resolution of module paths. r?ochameau draft
authorKris Maglione <maglione.k@gmail.com>
Wed, 12 Oct 2016 03:20:25 +0100
changeset 424078 7baac85b9656c9cb706d163e807bcc3584d9ff74
parent 424077 401bbfc5870159c164df134f1503dbc4f14c05b7
child 533586 98ec3423317a7030db5fd9cad32b9c8bb73cae41
push id32056
push usermaglione.k@gmail.com
push dateWed, 12 Oct 2016 04:31:54 +0000
reviewersochameau
bugs1309351
milestone52.0a1
Bug 1309351: Part 2 - Speed up synchronous resolution of module paths. r?ochameau This results in about a 28% speed-up for Jetpack mochitest runs, for me. MozReview-Commit-ID: K30q7BfvTLs
addon-sdk/source/lib/toolkit/loader.js
--- a/addon-sdk/source/lib/toolkit/loader.js
+++ b/addon-sdk/source/lib/toolkit/loader.js
@@ -23,22 +23,28 @@ module.metadata = {
   "stability": "unstable"
 };
 
 const { classes: Cc, Constructor: CC, interfaces: Ci, utils: Cu,
         results: Cr, manager: Cm } = Components;
 const systemPrincipal = CC('@mozilla.org/systemprincipal;1', 'nsIPrincipal')();
 const { loadSubScript } = Cc['@mozilla.org/moz/jssubscript-loader;1'].
                      getService(Ci.mozIJSSubScriptLoader);
-const { notifyObservers } = Cc['@mozilla.org/observer-service;1'].
+const { addObserver, notifyObservers } = Cc['@mozilla.org/observer-service;1'].
                         getService(Ci.nsIObserverService);
 const { XPCOMUtils } = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
 const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
 const { join: pathJoin, normalize, dirname } = Cu.import("resource://gre/modules/osfile/ospath_unix.jsm");
 
+XPCOMUtils.defineLazyServiceGetter(this, "resProto",
+                                   "@mozilla.org/network/protocol;1?name=resource",
+                                   "nsIResProtocolHandler");
+
+const ZipReader = CC("@mozilla.org/libjar/zip-reader;1", "nsIZipReader", "open");
+
 XPCOMUtils.defineLazyGetter(this, "XulApp", () => {
   let xulappURI = module.uri.replace("toolkit/loader.js",
                                      "sdk/system/xul-app.jsm");
   return Cu.import(xulappURI, {});
 });
 
 // Define some shortcuts.
 const bind = Function.call.bind(Function.bind);
@@ -197,24 +203,151 @@ function serializeStack(frames) {
            frame.fileName + ":" +
            frame.lineNumber + ":" +
            frame.columnNumber + "\n" +
            stack;
   }, "");
 }
 Loader.serializeStack = serializeStack;
 
+/**
+ * Returns a list of fully-qualified URLs for entries within the zip
+ * file at the given URI which are either directories or files with a
+ * .js or .json extension.
+ *
+ * @param {nsIJARURI} uri
+ * @param {string} baseURL
+ *        The original base URL, prior to resolution.
+ *
+ * @returns {Set<string>}
+ */
+function getZipFileContents(uri, baseURL) {
+  let file = uri.JARFile.QueryInterface(Ci.nsIFileURL).file;
+  let basePath = addTrailingSlash(uri.JAREntry).slice(1);
+
+  let zipReader = new ZipReader(file);
+  try {
+    let results = new Set();
+
+    let enumerator = zipReader.findEntries("(*.js|*.json|*/)");
+    while (enumerator.hasMore()) {
+      let entry = enumerator.getNext();
+      if (entry.startsWith(basePath)) {
+        let path = entry.slice(basePath.length);
+
+        results.add(baseURL + path);
+      }
+    }
+
+    return results;
+  } finally {
+    zipReader.close();
+  }
+}
+
+class DefaultMap extends Map {
+  constructor(createItem, items = undefined) {
+    super(items);
+
+    this.createItem = createItem;
+  }
+
+  get(key) {
+    if (!this.has(key))
+      this.set(key, this.createItem(key));
+
+    return super.get(key);
+  }
+}
+
+const zipContentsCache = new DefaultMap(baseURL => {
+  let uri = NetUtil.newURI(baseURL);
+
+  if (baseURL.startsWith("resource:"))
+    uri = NetUtil.newURI(resProto.resolveURI(uri));
+
+  if (uri instanceof Ci.nsIJARURI)
+    return getZipFileContents(uri, baseURL);
+
+  return null;
+});
+
+const filesCache = new DefaultMap(baseURL => {
+  return new DefaultMap(url => {
+    let uri = NetUtil.newURI(url);
+
+    try {
+      if (uri instanceof Ci.nsIFileURL)
+        return uri.file.exists();
+    } catch (e) {
+      // Throws for resource: URLs that are backed by jar: URLs.
+    }
+
+    return false;
+  });
+});
+
+// Clear any module resolution caches when the startup cache is flushed,
+// since it probably means we're loading new copies of extensions.
+function clearCaches() {
+  zipContentsCache.clear();
+  filesCache.clear();
+}
+
+let cacheObserver = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupportsWeakReference]),
+  observe: clearCaches,
+};
+addObserver(cacheObserver, "startupcache-invalidate", true);
+
+/**
+ * Returns the base URL for the given URL, if one can be determined. For
+ * a resource: URL, this is the root of the resource package. For a jar:
+ * URL, it is the root of the JAR file. Otherwise, null is returned.
+ *
+ * @param {string} url
+ * @returns {string?}
+ */
+function getBaseURL(url) {
+  if (url.startsWith("resource://"))
+    return /^(resource:\/\/[^\/]+\/)/.exec(url)[1];
+
+  let uri = NetUtil.newURI(url);
+  if (uri instanceof Ci.nsIJARURI)
+    return `jar:${uri.JARFile.spec}!/`;
+
+  return null;
+}
+
+/**
+ * Returns true if the target of the given URL exists as a local file,
+ * or as an entry in a local zip file.
+ *
+ * @param {string} url
+ * @returns {boolean}
+ */
+function checkUrlExists(url) {
+  if (!/\.(?:js|json)$/.test(url))
+    url = addTrailingSlash(url);
+
+  let baseURL = getBaseURL(url);
+
+  let scripts = zipContentsCache.get(baseURL);
+  if (scripts)
+    return scripts.has(url);
+
+  return filesCache.get(baseURL).get(url);
+}
+
 function readURI(uri) {
   let nsURI = NetUtil.newURI(uri);
   if (nsURI.scheme == "resource") {
     // Resolve to a real URI, this will catch any obvious bad paths without
     // logging assertions in debug builds, see bug 1135219
-    let proto = Cc["@mozilla.org/network/protocol;1?name=resource"].
-                getService(Ci.nsIResProtocolHandler);
-    uri = proto.resolveURI(nsURI);
+    uri = resProto.resolveURI(nsURI);
   }
 
   let stream = NetUtil.newChannel({
     uri: NetUtil.newURI(uri, 'UTF-8'),
     loadUsingSystemPrincipal: true}
   ).open2();
   let count = stream.available();
   let data = NetUtil.readInputStreamToString(stream, count, {
@@ -454,83 +587,70 @@ const resolve = iced(function resolve(id
 
   return resolved;
 });
 Loader.resolve = resolve;
 
 // Attempts to load `path` and then `path.js`
 // Returns `path` with valid file, or `undefined` otherwise
 function resolveAsFile(path) {
-  let found;
+  // Append '.js' to path name unless it's another support filetype
+  path = normalizeExt(path);
+  if (checkUrlExists(path))
+    return path;
 
-  // As per node's loader spec,
-  // we first should try and load 'path' (with no extension)
-  // before trying 'path.js'. We will not support this feature
-  // due to performance, but may add it if necessary for adoption.
-  try {
-    // Append '.js' to path name unless it's another support filetype
-    path = normalizeExt(path);
-    readURI(path);
-    found = path;
-  } catch (e) {}
-
-  return found;
+  return null;
 }
 
 // Attempts to load `path/package.json`'s `main` entry,
 // followed by `path/index.js`, or `undefined` otherwise
 function resolveAsDirectory(path) {
   try {
     // If `path/package.json` exists, parse the `main` entry
     // and attempt to load that
-    let main = getManifestMain(JSON.parse(readURI(path + '/package.json')));
-    if (main != null) {
-      let tmpPath = join(path, main);
-      let found = resolveAsFile(tmpPath);
+    let manifestPath = path + '/package.json';
+
+    let main = (checkUrlExists(manifestPath) &&
+                getManifestMain(JSON.parse(readURI(manifestPath))));
+    if (main) {
+      let found = resolveAsFile(join(path, main));
       if (found)
         return found
     }
   } catch (e) {}
 
-  try {
-    let tmpPath = path + '/index.js';
-    readURI(tmpPath);
-    return tmpPath;
-  } catch (e) {}
-
-  return null;
+  return resolveAsFile(path + '/index.js');
 }
 
 function resolveRelative(rootURI, modulesDir, id) {
   let fullId = join(rootURI, modulesDir, id);
-  let resolvedPath;
 
-  if ((resolvedPath = resolveAsFile(fullId)))
-    return stripBase(rootURI, resolvedPath);
-
-  if ((resolvedPath = resolveAsDirectory(fullId)))
+  let resolvedPath = (resolveAsFile(fullId) ||
+                      resolveAsDirectory(fullId));
+  if (resolvedPath)
     return stripBase(rootURI, resolvedPath);
 
   return null;
 }
 
 // From `resolve` module
 // https://github.com/substack/node-resolve/blob/master/lib/node-modules-paths.js
-function* getNodeModulePaths(start) {
-  // Configurable in node -- do we need this to be configurable?
+function* getNodeModulePaths(rootURI, start) {
   let moduleDir = 'node_modules';
 
   let parts = start.split('/');
   while (parts.length) {
     let leaf = parts.pop();
-    if (leaf !== moduleDir)
-      yield join(...parts, leaf, moduleDir);
+    let path = join(...parts, leaf, moduleDir);
+    if (leaf !== moduleDir && checkUrlExists(join(rootURI, path)))
+      yield path;
   }
 
-  yield moduleDir;
+  if (checkUrlExists(join(rootURI, moduleDir)))
+    yield moduleDir;
 }
 
 // Node-style module lookup
 // Takes an id and path and attempts to load a file using node's resolving
 // algorithm.
 // `id` should already be resolved relatively at this point.
 // http://nodejs.org/api/modules.html#modules_all_together
 const nodeResolve = iced(function nodeResolve(id, requirer, { rootURI }) {
@@ -550,17 +670,17 @@ const nodeResolve = iced(function nodeRe
 
   // If the requirer is an absolute URI then the node module resolution below
   // won't work correctly as we prefix everything with rootURI
   if (isAbsoluteURI(requirer))
     return null;
 
   // If manifest has dependencies, attempt to look up node modules
   // in the `dependencies` list
-  for (let modulesDir of getNodeModulePaths(dirname(requirer))) {
+  for (let modulesDir of getNodeModulePaths(rootURI, dirname(requirer))) {
     if ((resolvedPath = resolveRelative(rootURI, modulesDir, id)))
       return resolvedPath;
   }
 
   // We would not find lookup for things like `sdk/tabs`, as that's part of
   // the alias mapping. If during `generateMap`, the runtime lookup resolves
   // with `resolveURI` -- if during runtime, then `resolve` will throw.
   return null;