Bug 1434374 - rewrite scripts draft
authorAlexandre Poirot <poirot.alex@gmail.com>
Wed, 07 Feb 2018 08:06:24 -0800
changeset 752134 b1373c0967d4763c3336a514e11db992f5f7ac02
parent 752133 c19b070a728a104d69d2a0fd2930010e447f4c60
push id98173
push userbmo:poirot.alex@gmail.com
push dateWed, 07 Feb 2018 16:10:04 +0000
bugs1434374
milestone60.0a1
Bug 1434374 - rewrite scripts MozReview-Commit-ID: DLQ1OhnpThG
rewrite/Processor.jsm
rewrite/Replacer.jsm
rewrite/main.js
rewrite/processors/chromeutils-import-devtools.jsm
rewrite/run-replacer.sh
new file mode 100644
--- /dev/null
+++ b/rewrite/Processor.jsm
@@ -0,0 +1,168 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://gre/modules/osfile.jsm");
+ChromeUtils.import("resource://gre/modules/Subprocess.jsm");
+
+ChromeUtils.import("resource://rewrites/Replacer.jsm");
+
+
+Cc["@mozilla.org/jsreflect;1"].createInstance()();
+this.Reflect = Reflect;
+
+var EXPORTED_SYMBOLS = ["ProcessorBase", "Reflect"];
+
+async function readAll(pipe) {
+  let result = "";
+  let str;
+  while ((str = await pipe.readString())) {
+    result += str;
+  }
+  return result;
+}
+
+const EXTS = [".js", ".jsm",
+              ".xml", ".xhtml",
+              ".html", ".xul"];
+
+class ProcessorBase {
+  constructor(ignoredPaths = [], includePaths = null) {
+    this.ignoredPaths = ignoredPaths;
+    this.includePaths = includePaths;
+  }
+
+  async getTrackedFiles() {
+    let hg = await Subprocess.pathSearch("hg");
+    let proc = await Subprocess.call({
+        command: hg,
+        arguments: ["manifest"],
+        stdout: "pipe",
+      });
+    let files = await readAll(proc.stdout);
+    await proc.wait();
+    if (!files) {
+      dump("go git\n");
+      let git = await Subprocess.pathSearch("git");
+      proc = await Subprocess.call({
+        command: git,
+        arguments: ["ls-tree", "-r", "HEAD", "--name-only"],
+        stdout: "pipe",
+      });
+      files = await readAll(proc.stdout);
+      await proc.wait();
+    }
+    return files.split("\n");
+  }
+
+  shouldProcess(path, text) {
+    return true;
+  }
+
+  processScript(file, text) {
+    if (!this.shouldProcess(file, text)) {
+      return null;
+    }
+    let replacer = new Replacer(text, {preprocessor: true});
+    this.process(file, replacer);
+    return replacer.applyChanges();
+  }
+
+  processRegExp(file, text, pattern,
+                mangle = text => text,
+                demangle = text => text) {
+    let found = false;
+    let result = text.replace(pattern, (m0, m1, m2, m3) => {
+      if (m2) {
+        try {
+          let res = this.processScript(file, mangle(m2));
+          if (res) {
+            found = true;
+            return `${m1}${demangle(res)}${m3}`;
+          }
+        } catch (e) {
+          dump(`FAILED TO PROCESS PART OF FILE ${file}\n`);
+          dump(`Error: ${e}\n${e.stack}\n`);
+        }
+      }
+      return m0;
+    });
+
+    if (found) {
+      return result;
+    }
+    return null;
+  }
+
+  processHTML(file, text) {
+    return this.processRegExp(file, text, /(<script[^>]*>)([^]*?)(<\/script>)/g);
+  }
+
+  processXML(file, text) {
+    return this.processRegExp(file, text, /(<!\[CDATA\[)([^]*?)(\]\]>)/g);
+  }
+
+  processXBL(file, text) {
+    const PRE = "(function(){\n";
+    const POST = "\n})()";
+
+    function mangle(text) {
+      return `${PRE}${text}${POST}`;
+    }
+    function demangle(text) {
+      return text.slice(PRE.length, -POST.length);
+    }
+
+    return this.processRegExp(file, text, /(<!\[CDATA\[)([^]*?)(\]\]>)/g,
+                              mangle, demangle);
+  }
+
+  async processFiles() {
+    for (let path of await this.getTrackedFiles()) {
+      if (this.ignoredPaths.some(p => path.startsWith(p))) {
+        continue;
+      }
+      if (this.includePaths && !this.includePaths.some(p => path.startsWith(p))) {
+        continue;
+      }
+      try {
+        await this.processFile(path);
+      } catch (e) {
+        dump(`FAILED TO PROCESS FILE ${path}\n`);
+        dump(`Error: ${e}\n${e.stack}\n`);
+      }
+    }
+  }
+
+  async processFile(path) {
+    if (!EXTS.some(ext => path.endsWith(ext))) {
+      return;
+    }
+
+    let contents = new TextDecoder().decode(await OS.File.read(path));
+
+    if (!this.shouldProcess(path, contents)) {
+      return;
+    }
+
+    dump(`PROCESSING ${path}\n`);
+
+    let result;
+    if (path.endsWith(".js") || path.endsWith(".jsm")) {
+      result = this.processScript(path, contents);
+    } else if (path.endsWith(".html")) {
+      result = this.processHTML(path, contents);
+    } else if (path.endsWith(".xml")) {
+      if (contents.includes("xmlns=\"http://www.mozilla.org/xbl\"")) {
+        result = this.processXBL(path, contents);
+      }
+    } else {
+      result = this.processXML(path, contents);
+    }
+
+    if (result != null && result != contents) {
+      dump(`UPDATING ${path}\n`);
+      await OS.File.writeAtomic(path, new TextEncoder().encode(result));
+    }
+  }
+}
new file mode 100644
--- /dev/null
+++ b/rewrite/Replacer.jsm
@@ -0,0 +1,164 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+var EXPORTED_SYMBOLS = ["Replacer"];
+
+const PLACEHOLDER = "//SourceRewriterPreprocessorMacro-#";
+
+class Replacer {
+  constructor(code, {preprocessor = false} = {}) {
+    this.code = code;
+
+    if (preprocessor) {
+      this.manglePreprocessor();
+    }
+
+    this.lineOffsets = [0];
+    let re = /.*?\n|.+?$/g;
+    while (re.exec(code)) {
+      this.lineOffsets.push(re.lastIndex);
+    }
+
+    this.replacements = [];
+
+    this.preprocessor = preprocessor;
+  }
+
+  manglePreprocessor() {
+    this.code = this.code.replace(/^#/gm, PLACEHOLDER);
+  }
+
+  demanglePreprocessor(result) {
+    if (this.preprocessor) {
+      return result.replace(RegExp("^" + PLACEHOLDER, "gm"),
+                            "#");
+    }
+    return result;
+  }
+
+  applyChanges() {
+    this.replacements.sort((a, b) => a.start - b.start || b.end - a.end);
+
+    let parts = [];
+    let offset = 0;
+    let fillGap = end => {
+      if (offset < end) {
+        parts.push(this.code.slice(offset, end));
+      }
+      offset = end;
+    };
+
+    for (let {offsets, text} of this.replacements) {
+      if (offsets.start < offset) {
+        continue;
+      }
+
+      fillGap(offsets.start);
+      parts.push(text);
+
+      offset = offsets.end;
+    }
+    fillGap(this.code.length);
+
+    return this.demanglePreprocessor(parts.join(""));
+  }
+
+  getOffset(loc) {
+    return this.lineOffsets[loc.line - 1] + loc.column;
+  }
+
+  getOffsets(range) {
+    return {start: this.getOffset(range.start), end: this.getOffset(range.end)};
+  }
+
+  getArgOffsets(node) {
+    let args = node.arguments;
+    if (args.length) {
+      return this.getOffsets({start: args[0].loc.start,
+                              end: args[args.length - 1].loc.end});
+    }
+
+    let start = this.getOffset(node.callee.loc.end) + 1;
+    let end = this.getOffset(node.loc.end);
+
+    let text = this.code.slice(start, end + 1);
+    return {
+      start: start + text.indexOf("(") + 1,
+      end: start + text.lastIndexOf(")") - 1,
+    };
+  }
+
+  getArgText(node) {
+    return this.getText(this.getArgOffsets(node));
+  }
+
+  getArg(node, index) {
+    let args = node.arguments;
+    let offsets = this.getOffsets({start: args[index].loc.start,
+                                   end: args[index].loc.end});
+    return this.getText(offsets);
+  }
+
+  getText(offsets) {
+    return this.code.slice(offsets.start, offsets.end);
+  }
+
+  getNodeText(range) {
+    return this.getText(this.getOffsets(node.loc));
+  }
+
+  replace(node, text) {
+    this.replaceOffsets(this.getOffsets(node.loc),
+                        text);
+  }
+
+  replaceArgs(node, text) {
+    this.replaceOffsets(this.getArgOffsets(node), text);
+  }
+
+  replaceOffsets(offsets, text) {
+    this.replacements.push({
+      offsets,
+      text
+    });
+  }
+
+  replaceCallee(node, callee, args = null) {
+    if (callee.includes("\n")) {
+      throw new Error("Multi-line call expressions not supported");
+    }
+
+    this.replace(node.callee, callee);
+
+    let args_ = node.arguments;
+    if (!args && (!args_.length ||
+                  args_[0].loc.start.line === args_[args_.length - 1].loc.end.line)) {
+      return;
+    }
+    if (!args) {
+      args = this.getArgText(node);
+    }
+
+    let origIndent;
+    if (node.arguments.length &&
+        node.arguments[0].loc.start.line === node.callee.loc.end.line) {
+      origIndent = node.arguments[0].loc.start.column;
+    } else {
+      origIndent = node.callee.loc.end.column + 1;
+    }
+
+    let delta = (node.callee.loc.start.column + callee.length -
+                 node.callee.loc.end.column);
+    let newIndent = origIndent + delta;
+
+    let origSpaces = " ".repeat(origIndent);
+    let newSpaces = " ".repeat(newIndent);
+
+    args = args.replace(RegExp("\n" + origSpaces, "g"),
+                        "\n" + newSpaces);
+
+    this.replaceArgs(node, args);
+  }
+}
+
new file mode 100644
--- /dev/null
+++ b/rewrite/main.js
@@ -0,0 +1,34 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+const nsFile = Components.Constructor("@mozilla.org/file/local;1",
+                                      "nsIFile", "initWithPath");
+
+const resProto = Services.io.getProtocolHandler("resource")
+                         .QueryInterface(Ci.nsIResProtocolHandler);
+
+const baseDir = nsFile(arguments[0]);
+resProto.setSubstitution("rewrites", Services.io.newFileURI(baseDir));
+
+
+async function main(scriptdir, script, ...args) {
+  let temp = {};
+  ChromeUtils.import(`resource://rewrites/processors/${script}.jsm`, temp);
+
+  let processor = new temp.Processor(args);
+  await processor.processFiles();
+}
+
+let done = false;
+main(...arguments).catch(e => {
+  dump(`Error: ${e}\n${e.stack}\n`);
+}).then(() => {
+  done = true;
+});
+
+// Spin an event loop.
+Services.tm.spinEventLoopUntil(() => done);
+Services.tm.spinEventLoopUntilEmpty();
new file mode 100644
--- /dev/null
+++ b/rewrite/processors/chromeutils-import-devtools.jsm
@@ -0,0 +1,109 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://rewrites/Processor.jsm");
+
+var EXPORTED_SYMBOLS = ["Processor"];
+
+
+const kIgnorePaths = [];
+
+
+function isIdentifier(node, id) {
+  return node && node.type === "Identifier" && node.name === id;
+}
+
+function isMemberExpression(node, object, member) {
+  return (node && node.type === "MemberExpression" &&
+          isIdentifier(node.object, object) &&
+          isIdentifier(node.property, member));
+}
+
+const devtoolsJSMs = [
+  "browser-loader.js",
+  "base-loader.js",
+  "deprecated-sync-thenables.js",
+  "converter-observer.js",
+  "devtools-startup.js",
+  "aboutdevtoolstoolbox-registration.js",
+  "aboutdebugging-registration.js",
+  "aboutdevtools-registration.js",
+  "content-process-debugger-server.js",
+  "content-process-forward.js",
+  "loader.js",
+  "dom/main.js",
+
+  // Also list top level documents that don't necessarely import require
+  "webide/content",
+  "aboutdevtools",
+
+  // external modules
+  "httpd.js",
+]
+
+class Processor extends ProcessorBase {
+  constructor(filters) {
+    super(kIgnorePaths, filters.length ? filters : null);
+  }
+
+  shouldProcess(path, text) {
+    return /\b(?:Cu\.import\()/.test(text);
+  }
+
+  process(path, replacer) {
+    Reflect.parse(replacer.code, {
+      source: path,
+      builder: {
+        callExpression: function(callee, args, loc) {
+          let node = {type: "CallExpression", callee, arguments: args, loc};
+
+          let isCu = isIdentifier(callee.object, "Cu");
+
+          function shouldUseChromeUtils() {
+            let offsets = replacer.getOffsets(loc);
+            offsets.start = 0;
+            let text = replacer.getText(offsets);
+            return text.includes("Components") && text.includes("Cu") && !text.includes("require\(");
+          }
+          if (isCu && isIdentifier(callee.property, "import") && args.length >= 1) {
+            let arg1 = replacer.getArg(node, 0);
+
+            // Replace Cu.import(".../xxx.jsm"...) to ChromeUtils.import(".../xxx.jsm"...)
+            // * For Loader.jsm import, as that's the one implementing require.
+            // * For all Cu.import done for DevTools JSM files, devtoolsJSMs lists all of them.
+            //   We can't switch them to require() as 'require' would consider them as CommonJS modules.
+            // * For all Cu.import done from JSM files (there is devtoolsJSMs for jsm that don't have jsm extension)
+            // * Also keep ChromeUtils.import for all tests as they aren't CommonJS modules
+            //   and do not necessarely have access to require.
+            let { source } = loc;
+            if (source.includes("/test") || source.includes(".jsm") || arg1.includes("Loader.jsm") ||
+                devtoolsJSMs.some(jsm => arg1.includes("/" + jsm) || source.includes("/" + jsm))) {
+              replacer.replaceCallee(node, "ChromeUtils.import");
+            }
+
+            // Replace Cu.import("xxx.jsm", {}) to require("xxx.jsm")
+            else if (args.length == 2) {
+              let arg2 = replacer.getArg(node, 1)
+              if (arg2 == "{}") {
+                replacer.replaceCallee(node, "require", arg1);
+              } else {
+                dump("second arg text: "+arg2+"\n");
+                // When second arg isn't "{}", keep using ChromeUtils.import()
+                replacer.replaceCallee(node, "ChromeUtils.import");
+              }
+            }
+
+            // Replace Cu.import("xxx.jsm") to require("xxx.jsm")
+            else if (args.length == 1) {
+              replacer.replaceCallee(node, "require");
+            }
+          }
+
+          return node;
+        },
+      },
+    });
+  }
+}
+
new file mode 100755
--- /dev/null
+++ b/rewrite/run-replacer.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+scriptdir=$(readlink -f $(dirname $(which $0)))
+
+
+objdir=$(./mach environment | awk '$0 == "object directory:" { found = 1; next }; found { print $1; exit }')
+dist_bin="$objdir/dist/bin"
+
+export LD_LIBRARY_PATH="$dist_bin";
+export XPCSHELL="$dist_bin/xpcshell"
+
+"$XPCSHELL" "$scriptdir/main.js" "$scriptdir" "$@"