Bug 1434374 - rewrite scripts
MozReview-Commit-ID: DLQ1OhnpThG
new file mode 100644
--- /dev/null
+++ b/rewrite/Processor.jsm
@@ -0,0 +1,168 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://gre/modules/osfile.jsm");
+ChromeUtils.import("resource://gre/modules/Subprocess.jsm");
+
+ChromeUtils.import("resource://rewrites/Replacer.jsm");
+
+
+Cc["@mozilla.org/jsreflect;1"].createInstance()();
+this.Reflect = Reflect;
+
+var EXPORTED_SYMBOLS = ["ProcessorBase", "Reflect"];
+
+async function readAll(pipe) {
+ let result = "";
+ let str;
+ while ((str = await pipe.readString())) {
+ result += str;
+ }
+ return result;
+}
+
+const EXTS = [".js", ".jsm",
+ ".xml", ".xhtml",
+ ".html", ".xul"];
+
+class ProcessorBase {
+ constructor(ignoredPaths = [], includePaths = null) {
+ this.ignoredPaths = ignoredPaths;
+ this.includePaths = includePaths;
+ }
+
+ async getTrackedFiles() {
+ let hg = await Subprocess.pathSearch("hg");
+ let proc = await Subprocess.call({
+ command: hg,
+ arguments: ["manifest"],
+ stdout: "pipe",
+ });
+ let files = await readAll(proc.stdout);
+ await proc.wait();
+ if (!files) {
+ dump("go git\n");
+ let git = await Subprocess.pathSearch("git");
+ proc = await Subprocess.call({
+ command: git,
+ arguments: ["ls-tree", "-r", "HEAD", "--name-only"],
+ stdout: "pipe",
+ });
+ files = await readAll(proc.stdout);
+ await proc.wait();
+ }
+ return files.split("\n");
+ }
+
+ shouldProcess(path, text) {
+ return true;
+ }
+
+ processScript(file, text) {
+ if (!this.shouldProcess(file, text)) {
+ return null;
+ }
+ let replacer = new Replacer(text, {preprocessor: true});
+ this.process(file, replacer);
+ return replacer.applyChanges();
+ }
+
+ processRegExp(file, text, pattern,
+ mangle = text => text,
+ demangle = text => text) {
+ let found = false;
+ let result = text.replace(pattern, (m0, m1, m2, m3) => {
+ if (m2) {
+ try {
+ let res = this.processScript(file, mangle(m2));
+ if (res) {
+ found = true;
+ return `${m1}${demangle(res)}${m3}`;
+ }
+ } catch (e) {
+ dump(`FAILED TO PROCESS PART OF FILE ${file}\n`);
+ dump(`Error: ${e}\n${e.stack}\n`);
+ }
+ }
+ return m0;
+ });
+
+ if (found) {
+ return result;
+ }
+ return null;
+ }
+
+ processHTML(file, text) {
+ return this.processRegExp(file, text, /(<script[^>]*>)([^]*?)(<\/script>)/g);
+ }
+
+ processXML(file, text) {
+ return this.processRegExp(file, text, /(<!\[CDATA\[)([^]*?)(\]\]>)/g);
+ }
+
+ processXBL(file, text) {
+ const PRE = "(function(){\n";
+ const POST = "\n})()";
+
+ function mangle(text) {
+ return `${PRE}${text}${POST}`;
+ }
+ function demangle(text) {
+ return text.slice(PRE.length, -POST.length);
+ }
+
+ return this.processRegExp(file, text, /(<!\[CDATA\[)([^]*?)(\]\]>)/g,
+ mangle, demangle);
+ }
+
+ async processFiles() {
+ for (let path of await this.getTrackedFiles()) {
+ if (this.ignoredPaths.some(p => path.startsWith(p))) {
+ continue;
+ }
+ if (this.includePaths && !this.includePaths.some(p => path.startsWith(p))) {
+ continue;
+ }
+ try {
+ await this.processFile(path);
+ } catch (e) {
+ dump(`FAILED TO PROCESS FILE ${path}\n`);
+ dump(`Error: ${e}\n${e.stack}\n`);
+ }
+ }
+ }
+
+ async processFile(path) {
+ if (!EXTS.some(ext => path.endsWith(ext))) {
+ return;
+ }
+
+ let contents = new TextDecoder().decode(await OS.File.read(path));
+
+ if (!this.shouldProcess(path, contents)) {
+ return;
+ }
+
+ dump(`PROCESSING ${path}\n`);
+
+ let result;
+ if (path.endsWith(".js") || path.endsWith(".jsm")) {
+ result = this.processScript(path, contents);
+ } else if (path.endsWith(".html")) {
+ result = this.processHTML(path, contents);
+ } else if (path.endsWith(".xml")) {
+ if (contents.includes("xmlns=\"http://www.mozilla.org/xbl\"")) {
+ result = this.processXBL(path, contents);
+ }
+ } else {
+ result = this.processXML(path, contents);
+ }
+
+ if (result != null && result != contents) {
+ dump(`UPDATING ${path}\n`);
+ await OS.File.writeAtomic(path, new TextEncoder().encode(result));
+ }
+ }
+}
new file mode 100644
--- /dev/null
+++ b/rewrite/Replacer.jsm
@@ -0,0 +1,164 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+var EXPORTED_SYMBOLS = ["Replacer"];
+
+const PLACEHOLDER = "//SourceRewriterPreprocessorMacro-#";
+
+class Replacer {
+ constructor(code, {preprocessor = false} = {}) {
+ this.code = code;
+
+ if (preprocessor) {
+ this.manglePreprocessor();
+ }
+
+ this.lineOffsets = [0];
+ let re = /.*?\n|.+?$/g;
+ while (re.exec(code)) {
+ this.lineOffsets.push(re.lastIndex);
+ }
+
+ this.replacements = [];
+
+ this.preprocessor = preprocessor;
+ }
+
+ manglePreprocessor() {
+ this.code = this.code.replace(/^#/gm, PLACEHOLDER);
+ }
+
+ demanglePreprocessor(result) {
+ if (this.preprocessor) {
+ return result.replace(RegExp("^" + PLACEHOLDER, "gm"),
+ "#");
+ }
+ return result;
+ }
+
+ applyChanges() {
+ this.replacements.sort((a, b) => a.start - b.start || b.end - a.end);
+
+ let parts = [];
+ let offset = 0;
+ let fillGap = end => {
+ if (offset < end) {
+ parts.push(this.code.slice(offset, end));
+ }
+ offset = end;
+ };
+
+ for (let {offsets, text} of this.replacements) {
+ if (offsets.start < offset) {
+ continue;
+ }
+
+ fillGap(offsets.start);
+ parts.push(text);
+
+ offset = offsets.end;
+ }
+ fillGap(this.code.length);
+
+ return this.demanglePreprocessor(parts.join(""));
+ }
+
+ getOffset(loc) {
+ return this.lineOffsets[loc.line - 1] + loc.column;
+ }
+
+ getOffsets(range) {
+ return {start: this.getOffset(range.start), end: this.getOffset(range.end)};
+ }
+
+ getArgOffsets(node) {
+ let args = node.arguments;
+ if (args.length) {
+ return this.getOffsets({start: args[0].loc.start,
+ end: args[args.length - 1].loc.end});
+ }
+
+ let start = this.getOffset(node.callee.loc.end) + 1;
+ let end = this.getOffset(node.loc.end);
+
+ let text = this.code.slice(start, end + 1);
+ return {
+ start: start + text.indexOf("(") + 1,
+ end: start + text.lastIndexOf(")") - 1,
+ };
+ }
+
+ getArgText(node) {
+ return this.getText(this.getArgOffsets(node));
+ }
+
+ getArg(node, index) {
+ let args = node.arguments;
+ let offsets = this.getOffsets({start: args[index].loc.start,
+ end: args[index].loc.end});
+ return this.getText(offsets);
+ }
+
+ getText(offsets) {
+ return this.code.slice(offsets.start, offsets.end);
+ }
+
+ getNodeText(range) {
+ return this.getText(this.getOffsets(node.loc));
+ }
+
+ replace(node, text) {
+ this.replaceOffsets(this.getOffsets(node.loc),
+ text);
+ }
+
+ replaceArgs(node, text) {
+ this.replaceOffsets(this.getArgOffsets(node), text);
+ }
+
+ replaceOffsets(offsets, text) {
+ this.replacements.push({
+ offsets,
+ text
+ });
+ }
+
+ replaceCallee(node, callee, args = null) {
+ if (callee.includes("\n")) {
+ throw new Error("Multi-line call expressions not supported");
+ }
+
+ this.replace(node.callee, callee);
+
+ let args_ = node.arguments;
+ if (!args && (!args_.length ||
+ args_[0].loc.start.line === args_[args_.length - 1].loc.end.line)) {
+ return;
+ }
+ if (!args) {
+ args = this.getArgText(node);
+ }
+
+ let origIndent;
+ if (node.arguments.length &&
+ node.arguments[0].loc.start.line === node.callee.loc.end.line) {
+ origIndent = node.arguments[0].loc.start.column;
+ } else {
+ origIndent = node.callee.loc.end.column + 1;
+ }
+
+ let delta = (node.callee.loc.start.column + callee.length -
+ node.callee.loc.end.column);
+ let newIndent = origIndent + delta;
+
+ let origSpaces = " ".repeat(origIndent);
+ let newSpaces = " ".repeat(newIndent);
+
+ args = args.replace(RegExp("\n" + origSpaces, "g"),
+ "\n" + newSpaces);
+
+ this.replaceArgs(node, args);
+ }
+}
+
new file mode 100644
--- /dev/null
+++ b/rewrite/main.js
@@ -0,0 +1,34 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+const nsFile = Components.Constructor("@mozilla.org/file/local;1",
+ "nsIFile", "initWithPath");
+
+const resProto = Services.io.getProtocolHandler("resource")
+ .QueryInterface(Ci.nsIResProtocolHandler);
+
+const baseDir = nsFile(arguments[0]);
+resProto.setSubstitution("rewrites", Services.io.newFileURI(baseDir));
+
+
+async function main(scriptdir, script, ...args) {
+ let temp = {};
+ ChromeUtils.import(`resource://rewrites/processors/${script}.jsm`, temp);
+
+ let processor = new temp.Processor(args);
+ await processor.processFiles();
+}
+
+let done = false;
+main(...arguments).catch(e => {
+ dump(`Error: ${e}\n${e.stack}\n`);
+}).then(() => {
+ done = true;
+});
+
+// Spin an event loop.
+Services.tm.spinEventLoopUntil(() => done);
+Services.tm.spinEventLoopUntilEmpty();
new file mode 100644
--- /dev/null
+++ b/rewrite/processors/chromeutils-import-devtools.jsm
@@ -0,0 +1,109 @@
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+ChromeUtils.import("resource://rewrites/Processor.jsm");
+
+var EXPORTED_SYMBOLS = ["Processor"];
+
+
+const kIgnorePaths = [];
+
+
+function isIdentifier(node, id) {
+ return node && node.type === "Identifier" && node.name === id;
+}
+
+function isMemberExpression(node, object, member) {
+ return (node && node.type === "MemberExpression" &&
+ isIdentifier(node.object, object) &&
+ isIdentifier(node.property, member));
+}
+
+const devtoolsJSMs = [
+ "browser-loader.js",
+ "base-loader.js",
+ "deprecated-sync-thenables.js",
+ "converter-observer.js",
+ "devtools-startup.js",
+ "aboutdevtoolstoolbox-registration.js",
+ "aboutdebugging-registration.js",
+ "aboutdevtools-registration.js",
+ "content-process-debugger-server.js",
+ "content-process-forward.js",
+ "loader.js",
+ "dom/main.js",
+
+ // Also list top level documents that don't necessarely import require
+ "webide/content",
+ "aboutdevtools",
+
+ // external modules
+ "httpd.js",
+]
+
+class Processor extends ProcessorBase {
+ constructor(filters) {
+ super(kIgnorePaths, filters.length ? filters : null);
+ }
+
+ shouldProcess(path, text) {
+ return /\b(?:Cu\.import\()/.test(text);
+ }
+
+ process(path, replacer) {
+ Reflect.parse(replacer.code, {
+ source: path,
+ builder: {
+ callExpression: function(callee, args, loc) {
+ let node = {type: "CallExpression", callee, arguments: args, loc};
+
+ let isCu = isIdentifier(callee.object, "Cu");
+
+ function shouldUseChromeUtils() {
+ let offsets = replacer.getOffsets(loc);
+ offsets.start = 0;
+ let text = replacer.getText(offsets);
+ return text.includes("Components") && text.includes("Cu") && !text.includes("require\(");
+ }
+ if (isCu && isIdentifier(callee.property, "import") && args.length >= 1) {
+ let arg1 = replacer.getArg(node, 0);
+
+ // Replace Cu.import(".../xxx.jsm"...) to ChromeUtils.import(".../xxx.jsm"...)
+ // * For Loader.jsm import, as that's the one implementing require.
+ // * For all Cu.import done for DevTools JSM files, devtoolsJSMs lists all of them.
+ // We can't switch them to require() as 'require' would consider them as CommonJS modules.
+ // * For all Cu.import done from JSM files (there is devtoolsJSMs for jsm that don't have jsm extension)
+ // * Also keep ChromeUtils.import for all tests as they aren't CommonJS modules
+ // and do not necessarely have access to require.
+ let { source } = loc;
+ if (source.includes("/test") || source.includes(".jsm") || arg1.includes("Loader.jsm") ||
+ devtoolsJSMs.some(jsm => arg1.includes("/" + jsm) || source.includes("/" + jsm))) {
+ replacer.replaceCallee(node, "ChromeUtils.import");
+ }
+
+ // Replace Cu.import("xxx.jsm", {}) to require("xxx.jsm")
+ else if (args.length == 2) {
+ let arg2 = replacer.getArg(node, 1)
+ if (arg2 == "{}") {
+ replacer.replaceCallee(node, "require", arg1);
+ } else {
+ dump("second arg text: "+arg2+"\n");
+ // When second arg isn't "{}", keep using ChromeUtils.import()
+ replacer.replaceCallee(node, "ChromeUtils.import");
+ }
+ }
+
+ // Replace Cu.import("xxx.jsm") to require("xxx.jsm")
+ else if (args.length == 1) {
+ replacer.replaceCallee(node, "require");
+ }
+ }
+
+ return node;
+ },
+ },
+ });
+ }
+}
+
new file mode 100755
--- /dev/null
+++ b/rewrite/run-replacer.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+scriptdir=$(readlink -f $(dirname $(which $0)))
+
+
+objdir=$(./mach environment | awk '$0 == "object directory:" { found = 1; next }; found { print $1; exit }')
+dist_bin="$objdir/dist/bin"
+
+export LD_LIBRARY_PATH="$dist_bin";
+export XPCSHELL="$dist_bin/xpcshell"
+
+"$XPCSHELL" "$scriptdir/main.js" "$scriptdir" "$@"