Bug 1362786 - (2) Run NMParser in worker r?kmag draft
authorDoug Thayer <dothayer@mozilla.com>
Fri, 21 Jul 2017 14:14:17 -0700
changeset 613393 e1f1be5e49efedf1e78a5813af8d8789774deb08
parent 613392 7887b6c125788c017e8d8373cb42b105f0e81d38
child 613394 4d1fdafc23d35a0056e93570e9a4eecbff597a56
child 613395 5c49e0133e908a77004806f77495d2992c579f18
push id69780
push userbmo:dothayer@mozilla.com
push dateFri, 21 Jul 2017 21:21:35 +0000
reviewerskmag
bugs1362786
milestone56.0a1
Bug 1362786 - (2) Run NMParser in worker r?kmag Pulls out the NMParser work (parsing nm results and turning them into an ArrayBuffer'd map of addresses to symbols) into a worker. For OSX we will still need to do some work to run c++filt in the background, but this gets us most of the way there. Without a Subprocess.jsm usable from a worker, we'll have to bounce data back to the main thread in order to bounce it to the c++filt worker. MozReview-Commit-ID: LZi7J1qGpmh
browser/components/extensions/ParseBreakpadSymbols-worker.js
browser/components/extensions/ParseNMSymbols-worker.js
browser/components/extensions/ParseSymbols-worker.js
browser/components/extensions/ParseSymbols.jsm
browser/components/extensions/ext-geckoProfiler.js
browser/components/extensions/moz.build
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ParseBreakpadSymbols-worker.js
@@ -0,0 +1,58 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env worker */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm");
+importScripts("resource:///modules/ParseSymbols.jsm");
+
+async function fetchSymbolFile(url) {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(`got error status ${response.status}`);
+  }
+
+  return response.text();
+}
+
+function parse(text) {
+  const syms = new Map();
+
+  // Lines look like this:
+  //
+  // PUBLIC 3fc74 0 test_public_symbol
+  //
+  // FUNC 40330 8e 0 test_func_symbol
+  const symbolRegex = /\nPUBLIC ([0-9a-f]+) [0-9a-f]+ (.*)|\nFUNC ([0-9a-f]+) [0-9a-f]+ [0-9a-f]+ (.*)/g;
+
+  let match;
+  let approximateLength = 0;
+  while ((match = symbolRegex.exec(text))) {
+    const [address0, symbol0, address1, symbol1] = match.slice(1);
+    const address = parseInt(address0 || address1, 16);
+    const sym = (symbol0 || symbol1).trimRight();
+    syms.set(address, sym);
+    approximateLength += sym.length;
+  }
+
+  return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
+}
+
+onmessage = async e => {
+  try {
+    let text;
+    if (e.data.filepath) {
+      text = await OS.File.read(e.data.filepath, {encoding: "utf-8"});
+    } else if (e.data.url) {
+      text = await fetchSymbolFile(e.data.url);
+    }
+
+    const result = parse(text);
+    postMessage({result}, result.map(r => r.buffer));
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+  close();
+};
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ParseNMSymbols-worker.js
@@ -0,0 +1,85 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+/* eslint-env worker */
+
+"use strict";
+
+importScripts("resource:///modules/ParseSymbols.jsm");
+
+class WorkerNMParser {
+  constructor() {
+    this._decoder = new TextDecoder();
+    this._addrToSymMap = new Map();
+    this._approximateLength = 0;
+  }
+
+  consume(arrayBuffer) {
+    const data = this._decoder.decode(arrayBuffer, {stream: true});
+    const lineRegex = /.*\n?/g;
+    const buffer = this._currentLine + data;
+
+    let match;
+    while ((match = lineRegex.exec(buffer))) {
+      let [line] = match;
+      if (line[line.length - 1] === "\n") {
+        this._processLine(line);
+      } else {
+        this._currentLine = line;
+        break;
+      }
+    }
+  }
+
+  finish() {
+    this._processLine(this._currentLine);
+    return {syms: this._addrToSymMap, approximateLength: this._approximateLength};
+  }
+
+  _processLine(line) {
+    // Example lines:
+    // 00000000028c9888 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
+    // 0000000003a33730 r mozilla::OggDemuxer::~OggDemuxer()::{lambda()#1}::operator()() const::__func__
+    // 0000000003a33930 r mozilla::VPXDecoder::Drain()::{lambda()#1}::operator()() const::__func__
+    //
+    // Some lines have the form
+    // <address> ' ' <letter> ' ' <symbol>
+    // and some have the form
+    // <address> ' ' <symbol>
+    // The letter has a meaning, but we ignore it.
+
+    const regex = /([^ ]+) (?:. )?(.*)/;
+    let match = regex.exec(line);
+    if (match) {
+      const [, address, symbol] = match;
+      this._addrToSymMap.set(parseInt(address, 16), symbol);
+      this._approximateLength += symbol.length;
+    }
+  }
+}
+
+let nmParser;
+onmessage = async e => {
+  try {
+    if (!nmParser) {
+      nmParser = new WorkerNMParser();
+    }
+    if (e.data.finish) {
+      const {syms, approximateLength} = nmParser.finish();
+      let result;
+      if (e.data.isDarwin) {
+        result = ParseSymbols.convertSymsMapToDemanglerFormat(syms);
+      } else {
+        result = ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
+      }
+
+      postMessage({result}, result.map(r => r.buffer));
+      close();
+    } else {
+      nmParser.consume(e.data.buffer);
+    }
+  } catch (error) {
+    postMessage({error: error.toString()});
+  }
+};
deleted file mode 100644
--- a/browser/components/extensions/ParseSymbols-worker.js
+++ /dev/null
@@ -1,58 +0,0 @@
-/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
-/* vim: set sts=2 sw=2 et tw=80: */
-/* eslint-env worker */
-
-"use strict";
-
-importScripts("resource://gre/modules/osfile.jsm");
-importScripts("resource:///modules/ParseSymbols.jsm");
-
-async function fetchSymbolFile(url) {
-  const response = await fetch(url);
-
-  if (!response.ok) {
-    throw new Error(`got error status ${response.status}`);
-  }
-
-  return response.text();
-}
-
-function parse(text) {
-  const syms = new Map();
-
-  // Lines look like this:
-  //
-  // PUBLIC 3fc74 0 test_public_symbol
-  //
-  // FUNC 40330 8e 0 test_func_symbol
-  const symbolRegex = /\nPUBLIC ([0-9a-f]+) [0-9a-f]+ (.*)|\nFUNC ([0-9a-f]+) [0-9a-f]+ [0-9a-f]+ (.*)/g;
-
-  let match;
-  let approximateLength = 0;
-  while ((match = symbolRegex.exec(text))) {
-    const [address0, symbol0, address1, symbol1] = match.slice(1);
-    const address = parseInt(address0 || address1, 16);
-    const sym = (symbol0 || symbol1).trimRight();
-    syms.set(address, sym);
-    approximateLength += sym.length;
-  }
-
-  return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
-}
-
-onmessage = async e => {
-  try {
-    let text;
-    if (e.data.filepath) {
-      text = await OS.File.read(e.data.filepath, {encoding: "utf-8"});
-    } else if (e.data.url) {
-      text = await fetchSymbolFile(e.data.url);
-    }
-
-    const result = parse(text);
-    postMessage({result}, result.map(r => r.buffer));
-  } catch (error) {
-    postMessage({error: error.toString()});
-  }
-  close();
-};
--- a/browser/components/extensions/ParseSymbols.jsm
+++ b/browser/components/extensions/ParseSymbols.jsm
@@ -39,11 +39,23 @@ function convertSymsMapToExpectedSymForm
 
   const symsArray = addresses.map(addr => syms.get(addr));
   const {index, buffer} =
     convertStringArrayToUint8BufferWithIndex(symsArray, approximateSymLength);
 
   return [new Uint32Array(addresses), index, buffer];
 }
 
+function convertSymsMapToDemanglerFormat(syms) {
+  const addresses = Array.from(syms.keys());
+  addresses.sort((a, b) => a - b);
+
+  const symsArray = addresses.map(addr => syms.get(addr));
+  const textEncoder = new TextEncoder();
+  const buffer = textEncoder.encode(symsArray.join("\n"));
+
+  return [new Uint32Array(addresses), buffer];
+}
+
 var ParseSymbols = {
   convertSymsMapToExpectedSymFormat,
+  convertSymsMapToDemanglerFormat,
 };
--- a/browser/components/extensions/ext-geckoProfiler.js
+++ b/browser/components/extensions/ext-geckoProfiler.js
@@ -1,105 +1,83 @@
 /* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
 /* vim: set sts=2 sw=2 et tw=80: */
 "use strict";
 
 // The ext-* files are imported into the same scopes.
 /* import-globals-from ../../../toolkit/components/extensions/ext-toolkit.js */
 
 Cu.import("resource://gre/modules/Services.jsm");
-Cu.importGlobalProperties(["fetch", "TextEncoder"]);
+Cu.importGlobalProperties(["fetch", "TextEncoder", "TextDecoder"]);
 
 XPCOMUtils.defineLazyModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ParseSymbols", "resource:///modules/ParseSymbols.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Subprocess", "resource://gre/modules/Subprocess.jsm");
 
 const PREF_ASYNC_STACK = "javascript.options.asyncstack";
 const PREF_SYMBOLS_URL = "extensions.geckoProfiler.symbols.url";
 const PREF_GET_SYMBOL_RULES = "extensions.geckoProfiler.getSymbolRules";
 
 const ASYNC_STACKS_ENABLED = Services.prefs.getBoolPref(PREF_ASYNC_STACK, false);
 
 var {
   ExtensionError,
 } = ExtensionUtils;
 
 const parseSym = data => {
-  const worker = new ChromeWorker("resource://app/modules/ParseSymbols-worker.js");
+  const worker = new ChromeWorker("resource://app/modules/ParseBreakpadSymbols-worker.js");
   const promise = new Promise((resolve, reject) => {
     worker.onmessage = (e) => {
       if (e.data.error) {
         reject(e.data.error);
       } else {
         resolve(e.data.result);
       }
     };
   });
   worker.postMessage(data);
   return promise;
 };
 
 class NMParser {
   constructor() {
-    this._addrToSymMap = new Map();
-    this._approximateLength = 0;
+    this._worker = new ChromeWorker("resource://app/modules/ParseNMSymbols-worker.js");
   }
 
-  consume(data) {
-    const lineRegex = /.*\n?/g;
-    const buffer = this._currentLine + data;
-
-    let match;
-    while ((match = lineRegex.exec(buffer))) {
-      let [line] = match;
-      if (line[line.length - 1] === "\n") {
-        this._processLine(line);
-      } else {
-        this._currentLine = line;
-        break;
-      }
-    }
+  consume(buffer) {
+    this._worker.postMessage({buffer}, [buffer]);
   }
 
   finish() {
-    this._processLine(this._currentLine);
-    return {syms: this._addrToSymMap, approximateLength: this._approximateLength};
-  }
-
-  _processLine(line) {
-    // Example lines:
-    // 00000000028c9888 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 00000000028c9874 t GrFragmentProcessor::MulOutputByInputUnpremulColor(sk_sp<GrFragmentProcessor>)::PremulFragmentProcessor::onCreateGLSLInstance() const::GLFP::~GLFP()
-    // 0000000003a33730 r mozilla::OggDemuxer::~OggDemuxer()::{lambda()#1}::operator()() const::__func__
-    // 0000000003a33930 r mozilla::VPXDecoder::Drain()::{lambda()#1}::operator()() const::__func__
-    //
-    // Some lines have the form
-    // <address> ' ' <letter> ' ' <symbol>
-    // and some have the form
-    // <address> ' ' <symbol>
-    // The letter has a meaning, but we ignore it.
-
-    const regex = /([^ ]+) (?:. )?(.*)/;
-    let match = regex.exec(line);
-    if (match) {
-      const [, address, symbol] = match;
-      this._addrToSymMap.set(parseInt(address, 16), symbol);
-      this._approximateLength += symbol.length;
-    }
+    const promise = new Promise((resolve, reject) => {
+      this._worker.onmessage = (e) => {
+        if (e.data.error) {
+          reject(e.data.error);
+        } else {
+          resolve(e.data.result);
+        }
+      };
+    });
+    this._worker.postMessage({
+      finish: true,
+      isDarwin: Services.appinfo.OS === "Darwin",
+    });
+    return promise;
   }
 }
 
 class CppFiltParser {
   constructor(length) {
+    this._decoder = new TextDecoder();
     this._index = 0;
     this._results = new Array(length);
   }
 
-  consume(data) {
+  consume(arrayBuffer) {
+    const data = this._decoder.decode(arrayBuffer, {stream: true});
     const lineRegex = /.*\n?/g;
     const buffer = this._currentLine + data;
 
     let match;
     while ((match = lineRegex.exec(buffer))) {
       let [line] = match;
       if (line[line.length - 1] === "\n") {
         this._processLine(line);
@@ -117,17 +95,17 @@ class CppFiltParser {
 
   _processLine(line) {
     this._results[this._index++] = line.trimRight();
   }
 }
 
 const readAllData = async function(pipe, processData) {
   let data;
-  while ((data = await pipe.readString())) {
+  while ((data = await pipe.read()) && data.byteLength) {
     processData(data);
   }
 };
 
 const spawnProcess = async function(name, cmdArgs, processData, stdin = null) {
   const opts = {
     command: await Subprocess.pathSearch(name),
     arguments: cmdArgs,
@@ -152,29 +130,32 @@ const getSymbolsFromNM = async function(
   } else {
     // Mac's `nm` doesn't support the demangle option, so we have to
     // post-process the symbols with c++filt.
     args.unshift("--demangle");
   }
 
   await spawnProcess("nm", args, data => parser.consume(data));
   await spawnProcess("nm", ["-D", ...args], data => parser.consume(data));
-  let {syms, approximateLength} = parser.finish();
+  let result = await parser.finish();
+  if (Services.appinfo.OS !== "Darwin") {
+    return result;
+  }
 
-  if (Services.appinfo.OS === "Darwin") {
-    const keys = Array.from(syms.keys());
-    const values = keys.map(k => syms.get(k));
-    const demangler = new CppFiltParser(keys.length);
-    await spawnProcess("c++filt", [], data => demangler.consume(data), values.join("\n"));
-    const newSymbols = demangler.finish();
-    approximateLength = 0;
-    for (let [i, symbol] of newSymbols.entries()) {
-      approximateLength += symbol.length;
-      syms.set(keys[i], symbol);
-    }
+  const syms = new Map();
+  const [addresses, symbolsJoinedBuffer] = result;
+  const decoder = new TextDecoder();
+  const symbolsJoined = decoder.decode(symbolsJoinedBuffer);
+  const demangler = new CppFiltParser(addresses.length);
+  await spawnProcess("c++filt", [], data => demangler.consume(data), symbolsJoined);
+  const newSymbols = demangler.finish();
+  let approximateLength = 0;
+  for (let [i, symbol] of newSymbols.entries()) {
+    approximateLength += symbol.length;
+    syms.set(addresses[i], symbol);
   }
 
   return ParseSymbols.convertSymsMapToExpectedSymFormat(syms, approximateLength);
 };
 
 const pathComponentsForSymbolFile = (debugName, breakpadId) => {
   const symName = debugName.replace(/(\.pdb)?$/, ".sym");
   return [debugName, breakpadId, symName];
--- a/browser/components/extensions/moz.build
+++ b/browser/components/extensions/moz.build
@@ -10,17 +10,18 @@ with Files("**"):
 JAR_MANIFESTS += ['jar.mn']
 
 EXTRA_COMPONENTS += [
     'extensions-browser.manifest',
 ]
 
 EXTRA_JS_MODULES += [
     'ExtensionPopups.jsm',
-    'ParseSymbols-worker.js',
+    'ParseBreakpadSymbols-worker.js',
+    'ParseNMSymbols-worker.js',
     'ParseSymbols.jsm',
 ]
 
 DIRS += ['schemas']
 
 BROWSER_CHROME_MANIFESTS += [
     'test/browser/browser-remote.ini',
     'test/browser/browser.ini',