Bug 1311171 - Implement the devtools.network.onRequestFinished API event; r=rpl,jdescottes draft
authorJan Odvarko <odvarko@gmail.com>
Wed, 14 Feb 2018 11:32:10 +0100
changeset 754810 15607e8f725d88a5a94c9a5cf63b653566ad76f4
parent 754572 e43f2f6ea111c2d059d95fa9a71516b869a69698
push id99008
push userjodvarko@mozilla.com
push dateWed, 14 Feb 2018 11:10:38 +0000
reviewersrpl, jdescottes
bugs1311171
milestone60.0a1
Bug 1311171 - Implement the devtools.network.onRequestFinished API event; r=rpl,jdescottes MozReview-Commit-ID: IymuzcUg0VN
browser/components/extensions/ext-c-browser.js
browser/components/extensions/ext-c-devtools-network.js
browser/components/extensions/ext-devtools-network.js
browser/components/extensions/jar.mn
browser/components/extensions/schemas/devtools_network.json
browser/components/extensions/test/browser/browser_ext_devtools_network.js
devtools/client/framework/toolbox.js
devtools/client/netmonitor/initializer.js
devtools/client/netmonitor/src/har/har-exporter.js
--- a/browser/components/extensions/ext-c-browser.js
+++ b/browser/components/extensions/ext-c-browser.js
@@ -17,16 +17,23 @@ extensions.registerModules({
   },
   devtools_panels: {
     url: "chrome://browser/content/ext-c-devtools-panels.js",
     scopes: ["devtools_child"],
     paths: [
       ["devtools", "panels"],
     ],
   },
+  devtools_network: {
+    url: "chrome://browser/content/ext-c-devtools-network.js",
+    scopes: ["devtools_child"],
+    paths: [
+      ["devtools", "network"],
+    ],
+  },
   // Because of permissions, the module name must differ from both namespaces.
   menusInternal: {
     url: "chrome://browser/content/ext-c-menus.js",
     scopes: ["addon_child"],
     paths: [
       ["contextMenus"],
       ["menus"],
     ],
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/ext-c-devtools-network.js
@@ -0,0 +1,59 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+"use strict";
+
+// The ext-* files are imported into the same scopes.
+/* import-globals-from ../../../toolkit/components/extensions/ext-c-toolkit.js */
+
+/**
+ * Responsible for fetching HTTP response content from the backend.
+ *
+ * @param {DevtoolsExtensionContext}
+ *   A devtools extension context running in a child process.
+ * @param {object} options
+ */
+class ChildNetworkResponseLoader {
+  constructor(context, requestId) {
+    this.context = context;
+    this.requestId = requestId;
+  }
+
+  api() {
+    const {context, requestId} = this;
+    return {
+      getContent(callback) {
+        return context.childManager.callParentAsyncFunction(
+          "devtools.network.Request.getContent",
+          [requestId],
+          callback);
+      },
+    };
+  }
+}
+
+this.devtools_network = class extends ExtensionAPI {
+  getAPI(context) {
+    return {
+      devtools: {
+        network: {
+          onRequestFinished: new EventManager(context, "devtools.network.onRequestFinished", fire => {
+            let onFinished = (data) => {
+              const loader = new ChildNetworkResponseLoader(context, data.requestId);
+              const harEntry = {...data.harEntry, ...loader.api()};
+              const result = Cu.cloneInto(harEntry, context.cloneScope, {
+                cloneFunctions: true,
+              });
+              fire.asyncWithoutClone(result);
+            };
+
+            let parent = context.childManager.getParentEvent("devtools.network.onRequestFinished");
+            parent.addListener(onFinished);
+            return () => {
+              parent.removeListener(onFinished);
+            };
+          }).api(),
+        },
+      },
+    };
+  }
+};
--- a/browser/components/extensions/ext-devtools-network.js
+++ b/browser/components/extensions/ext-devtools-network.js
@@ -1,15 +1,19 @@
 /* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
 /* vim: set sts=2 sw=2 et tw=80: */
 "use strict";
 
 // The ext-* files are imported into the same scopes.
 /* import-globals-from ext-devtools.js */
 
+var {
+  SpreadArgs,
+} = ExtensionCommon;
+
 this.devtools_network = class extends ExtensionAPI {
   getAPI(context) {
     return {
       devtools: {
         network: {
           onNavigated: new EventManager(context, "devtools.onNavigated", fire => {
             let listener = (event, data) => {
               fire.async(data.url);
@@ -24,13 +28,42 @@ this.devtools_network = class extends Ex
                 target.off("navigate", listener);
               });
             };
           }).api(),
 
           getHAR: function() {
             return context.devToolsToolbox.getHARFromNetMonitor();
           },
+
+          onRequestFinished: new EventManager(context, "devtools.network.onRequestFinished", fire => {
+            const listener = (data) => {
+              fire.async(data);
+            };
+
+            const toolbox = context.devToolsToolbox;
+            toolbox.addRequestFinishedListener(listener);
+
+            return () => {
+              toolbox.removeRequestFinishedListener(listener);
+            };
+          }).api(),
+
+          // The following method is used internally to allow the request API
+          // piece that is running in the child process to ask the parent process
+          // to fetch response content from the back-end.
+          Request: {
+            async getContent(requestId) {
+              return context.devToolsToolbox.fetchResponseContent(requestId)
+                .then(({content}) => new SpreadArgs([content.text, content.mimeType]))
+                .catch(err => {
+                  const debugName = context.extension.policy.debugName;
+                  const errorMsg = "Unexpected error while fetching response content";
+                  Cu.reportError(`${debugName}: ${errorMsg} for ${requestId}: ${err}`);
+                  throw new ExtensionError(errorMsg);
+                });
+            },
+          },
         },
       },
     };
   }
 };
--- a/browser/components/extensions/jar.mn
+++ b/browser/components/extensions/jar.mn
@@ -32,13 +32,14 @@ browser.jar:
     content/browser/ext-pkcs11.js
     content/browser/ext-sessions.js
     content/browser/ext-sidebarAction.js
     content/browser/ext-tabs.js
     content/browser/ext-url-overrides.js
     content/browser/ext-windows.js
     content/browser/ext-c-browser.js
     content/browser/ext-c-devtools-inspectedWindow.js
+    content/browser/ext-c-devtools-network.js
     content/browser/ext-c-devtools-panels.js
     content/browser/ext-c-devtools.js
     content/browser/ext-c-menus.js
     content/browser/ext-c-omnibox.js
     content/browser/ext-c-tabs.js
--- a/browser/components/extensions/schemas/devtools_network.json
+++ b/browser/components/extensions/schemas/devtools_network.json
@@ -63,21 +63,24 @@
             ]
           }
         ]
       }
     ],
     "events": [
       {
         "name": "onRequestFinished",
-        "unsupported": true,
         "type": "function",
         "description": "Fired when a network request is finished and all request data are available.",
         "parameters": [
-          { "name": "request", "$ref": "Request", "description": "Description of a network request in the form of a HAR entry. See HAR specification for details." }
+          {
+            "name": "request",
+            "$ref": "Request",
+            "description": "Description of a network request in the form of a HAR entry. See HAR specification for details."
+          }
         ]
       },
       {
         "name": "onNavigated",
         "type": "function",
         "description": "Fired when the inspected window navigates to a new page.",
         "parameters": [
           {
--- a/browser/components/extensions/test/browser/browser_ext_devtools_network.js
+++ b/browser/components/extensions/test/browser/browser_ext_devtools_network.js
@@ -50,16 +50,38 @@ function devtools_page() {
     browser.test.sendMessage("getHAR-result", harLog);
 
     if (harLogCount === 2) {
       harLogCount = 0;
       browser.test.onMessage.removeListener(harListener);
     }
   };
   browser.test.onMessage.addListener(harListener);
+
+  let requestFinishedListener = async request => {
+    browser.test.assertTrue(request.request, "Request entry must exist");
+    browser.test.assertTrue(request.response, "Response entry must exist");
+
+    browser.test.sendMessage("onRequestFinished");
+
+    // Get response content using callback
+    request.getContent((content, encoding) => {
+      browser.test.sendMessage("onRequestFinished-callbackExecuted",
+                               [content, encoding]);
+    });
+
+    // Get response content using returned promise
+    request.getContent().then(([content, encoding]) => {
+      browser.test.sendMessage("onRequestFinished-promiseResolved",
+                               [content, encoding]);
+    });
+
+    browser.devtools.network.onRequestFinished.removeListener(requestFinishedListener);
+  };
+  browser.devtools.network.onRequestFinished.addListener(requestFinishedListener);
 }
 
 function waitForRequestAdded(toolbox) {
   return new Promise(resolve => {
     let netPanel = toolbox.getPanel("netmonitor");
     netPanel.panelWin.once("NetMonitor:RequestAdded", () => {
       resolve();
     });
@@ -152,16 +174,17 @@ add_task(async function test_devtools_ne
   // Reload the page to collect some HTTP requests.
   extension.sendMessage("navigate");
 
   // Wait till the navigation is complete and request
   // added into the net panel.
   await Promise.all([
     extension.awaitMessage("tabUpdated"),
     extension.awaitMessage("onNavigatedFired"),
+    extension.awaitMessage("onRequestFinished"),
     waitForRequestAdded(toolbox),
   ]);
 
   // Get HAR, it should not be empty now.
   const getHARPromise = extension.awaitMessage("getHAR-result");
   extension.sendMessage("getHAR");
   const getHARResult = await getHARPromise;
   is(getHARResult.log.entries.length, 1, "HAR log should not be empty");
@@ -170,8 +193,61 @@ add_task(async function test_devtools_ne
   await gDevTools.closeToolbox(target);
 
   await target.destroy();
 
   await extension.unload();
 
   await BrowserTestUtils.removeTab(tab);
 });
+
+/**
+ * Test for `chrome.devtools.network.onRequestFinished()` API
+ */
+add_task(async function test_devtools_network_on_request_finished() {
+  let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, "http://mochi.test:8888/");
+  let extension = ExtensionTestUtils.loadExtension(extData);
+
+  await extension.startup();
+  await extension.awaitMessage("ready");
+
+  let target = gDevTools.getTargetForTab(tab);
+
+  // Open the Toolbox
+  let toolbox = await gDevTools.showToolbox(target, "netmonitor");
+  info("Developer toolbox opened.");
+
+  // Reload and wait for onRequestFinished event.
+  extension.sendMessage("navigate");
+
+  await Promise.all([
+    extension.awaitMessage("tabUpdated"),
+    extension.awaitMessage("onNavigatedFired"),
+    waitForRequestAdded(toolbox),
+  ]);
+
+  await extension.awaitMessage("onRequestFinished");
+
+  // Wait for response content being fetched.
+  let [callbackRes, promiseRes] = await Promise.all([
+    extension.awaitMessage("onRequestFinished-callbackExecuted"),
+    extension.awaitMessage("onRequestFinished-promiseResolved"),
+  ]);
+
+  ok(callbackRes[0].startsWith("<html>"),
+     "The expected content has been retrieved.");
+  is(callbackRes[1], "text/html; charset=utf-8",
+     "The expected content has been retrieved.");
+
+  is(promiseRes[0], callbackRes[0],
+     "The resolved value is equal to the one received in the callback API mode");
+  is(promiseRes[1], callbackRes[1],
+     "The resolved value is equal to the one received in the callback API mode");
+
+  // Shutdown
+  await gDevTools.closeToolbox(target);
+
+  await target.destroy();
+
+  await extension.unload();
+
+  await BrowserTestUtils.removeTab(tab);
+});
--- a/devtools/client/framework/toolbox.js
+++ b/devtools/client/framework/toolbox.js
@@ -110,16 +110,19 @@ function Toolbox(target, selectedTool, h
   this._initInspector = null;
   this._inspector = null;
   this._styleSheets = null;
 
   // Map of frames (id => frame-info) and currently selected frame id.
   this.frameMap = new Map();
   this.selectedFrameId = null;
 
+  // List of listeners for `devtools.network.onRequestFinished` WebExt API
+  this._requestFinishedListeners = new Set();
+
   this._toolRegistered = this._toolRegistered.bind(this);
   this._toolUnregistered = this._toolUnregistered.bind(this);
   this._onWillNavigate = this._onWillNavigate.bind(this);
   this._refreshHostTitle = this._refreshHostTitle.bind(this);
   this._toggleNoAutohide = this._toggleNoAutohide.bind(this);
   this.showFramesMenu = this.showFramesMenu.bind(this);
   this.handleKeyDownOnFramesButton = this.handleKeyDownOnFramesButton.bind(this);
   this.showFramesMenuOnKeyDown = this.showFramesMenuOnKeyDown.bind(this);
@@ -2993,25 +2996,76 @@ Toolbox.prototype = {
   /**
    * Opens source in plain "view-source:".
    * @see devtools/client/shared/source-utils.js
    */
   viewSource: function (sourceURL, sourceLine) {
     return viewSource.viewSource(this, sourceURL, sourceLine);
   },
 
+  // Support for WebExtensions API (`devtools.network.*`)
+
   /**
    * Returns data (HAR) collected by the Network panel.
    */
   getHARFromNetMonitor: function () {
     let netPanel = this.getPanel("netmonitor");
 
     // The panel doesn't have to exist (it must be selected
     // by the user at least once to be created).
     // Return default empty HAR file in such case.
     if (!netPanel) {
       return Promise.resolve(buildHarLog(Services.appinfo));
     }
 
     // Use Netmonitor object to get the current HAR log.
     return netPanel.panelWin.Netmonitor.getHar();
+  },
+
+  /**
+   * Add listener for `onRequestFinished` events.
+   *
+   * @param {Object} listener
+   *        The listener to be called it's expected to be
+   *        a function that takes ({harEntry, requestId})
+   *        as first argument.
+   */
+  addRequestFinishedListener: function (listener) {
+    // Log console message informing the extension developer
+    // that the Network panel needs to be selected at least
+    // once in order to receive `onRequestFinished` events.
+    let message = "The Network panel needs to be selected at least" +
+      " once in order to receive 'onRequestFinished' events.";
+    this.target.logErrorInPage(message, "har");
+
+    // Add the listener into internal list.
+    this._requestFinishedListeners.add(listener);
+  },
+
+  removeRequestFinishedListener: function (listener) {
+    this._requestFinishedListeners.delete(listener);
+  },
+
+  getRequestFinishedListeners: function () {
+    return this._requestFinishedListeners;
+  },
+
+  /**
+   * Used to lazily fetch HTTP response content within
+   * `onRequestFinished` event listener.
+   *
+   * @param {String} requestId
+   *        Id of the request for which the response content
+   *        should be fetched.
+   */
+  fetchResponseContent: function (requestId) {
+    let netPanel = this.getPanel("netmonitor");
+
+    // The panel doesn't have to exist (it must be selected
+    // by the user at least once to be created).
+    // Return undefined content in such case.
+    if (!netPanel) {
+      return Promise.resolve({content: {}});
+    }
+
+    return netPanel.panelWin.Netmonitor.fetchResponseContent(requestId);
   }
 };
--- a/devtools/client/netmonitor/initializer.js
+++ b/devtools/client/netmonitor/initializer.js
@@ -45,48 +45,55 @@ window.connector = connector;
 /**
  * Global Netmonitor object in this panel. This object can be consumed
  * by other panels (e.g. Console is using inspectRequest), by the
  * Launchpad (bootstrap), WebExtension API (getHAR), etc.
  */
 window.Netmonitor = {
   bootstrap({ toolbox, panel }) {
     this.mount = document.querySelector("#mount");
+    this.toolbox = toolbox;
 
     const connection = {
       tabConnection: {
         tabTarget: toolbox.target,
       },
       toolbox,
       panel,
     };
 
     const openLink = (link) => {
       let parentDoc = toolbox.doc;
       let iframe = parentDoc.getElementById("toolbox-panel-iframe-netmonitor");
       let top = iframe.ownerDocument.defaultView.top;
       top.openUILinkIn(link, "tab");
     };
 
+    this.onRequestAdded = this.onRequestAdded.bind(this);
+    window.on(EVENTS.REQUEST_ADDED, this.onRequestAdded);
+
     // Render the root Application component.
     const sourceMapService = toolbox.sourceMapURLService;
     const app = App({ connector, openLink, sourceMapService });
     render(Provider({ store }, app), this.mount);
 
     // Connect to the Firefox backend by default.
     return connector.connectFirefox(connection, actions, store.getState);
   },
 
   destroy() {
     unmountComponentAtNode(this.mount);
+    window.off(EVENTS.REQUEST_ADDED, this.onRequestAdded);
     return connector.disconnect();
   },
 
+  // Support for WebExtensions API
+
   /**
-   * Returns list of requests currently available in the panel.
+   * Support for `devtools.network.getHAR` (get collected data as HAR)
    */
   getHar() {
     let { HarExporter } = require("devtools/client/netmonitor/src/har/har-exporter");
     let {
       getLongString,
       getTabTarget,
       getTimingMarker,
       requestData,
@@ -101,16 +108,56 @@ window.Netmonitor = {
       getTimingMarker,
       title: title || url,
     };
 
     return HarExporter.getHar(options);
   },
 
   /**
+   * Support for `devtools.network.onRequestFinished`. A hook for
+   * every finished HTTP request used by WebExtensions API.
+   */
+  onRequestAdded(event, requestId) {
+    let listeners = this.toolbox.getRequestFinishedListeners();
+    if (!listeners.size) {
+      return;
+    }
+
+    let { HarExporter } = require("devtools/client/netmonitor/src/har/har-exporter");
+    let { getLongString, getTabTarget, requestData } = connector;
+    let { form: { title, url } } = getTabTarget();
+
+    let options = {
+      getString: getLongString,
+      requestData,
+      title: title || url,
+      includeResponseBodies: false,
+      items: [getDisplayedRequestById(store.getState(), requestId)],
+    };
+
+    // Build HAR for specified request only.
+    HarExporter.getHar(options).then(har => {
+      let harEntry = har.log.entries[0];
+      delete harEntry.pageref;
+      listeners.forEach(listener => listener({
+        harEntry,
+        requestId,
+      }));
+    });
+  },
+
+  /**
+   * Support for `Request.getContent` WebExt API (lazy loading response body)
+   */
+  fetchResponseContent(requestId) {
+    return connector.requestData(requestId, "responseContent");
+  },
+
+  /**
    * Selects the specified request in the waterfall and opens the details view.
    * This is a firefox toolbox specific API, which providing an ability to inspect
    * a network request directly from other internal toolbox panel.
    *
    * @param {string} requestId The actor ID of the request to inspect.
    * @return {object} A promise resolved once the task finishes.
    */
   inspectRequest(requestId) {
--- a/devtools/client/netmonitor/src/har/har-exporter.js
+++ b/devtools/client/netmonitor/src/har/har-exporter.js
@@ -131,25 +131,32 @@ const HarExporter = {
 
   // Helpers
 
   fetchHarData: function (options) {
     // Generate page ID
     options.id = options.id || uid++;
 
     // Set default generic HAR export options.
-    options.jsonp = options.jsonp ||
-      Services.prefs.getBoolPref("devtools.netmonitor.har.jsonp");
-    options.includeResponseBodies = options.includeResponseBodies ||
-      Services.prefs.getBoolPref(
+    if (typeof options.jsonp != "boolean") {
+      options.jsonp = Services.prefs.getBoolPref(
+        "devtools.netmonitor.har.jsonp");
+    }
+    if (typeof options.includeResponseBodies != "boolean") {
+      options.includeResponseBodies = Services.prefs.getBoolPref(
         "devtools.netmonitor.har.includeResponseBodies");
-    options.jsonpCallback = options.jsonpCallback ||
-      Services.prefs.getCharPref("devtools.netmonitor.har.jsonpCallback");
-    options.forceExport = options.forceExport ||
-      Services.prefs.getBoolPref("devtools.netmonitor.har.forceExport");
+    }
+    if (typeof options.jsonpCallback != "boolean") {
+      options.jsonpCallback = Services.prefs.getCharPref(
+        "devtools.netmonitor.har.jsonpCallback");
+    }
+    if (typeof options.forceExport != "boolean") {
+      options.forceExport = Services.prefs.getBoolPref(
+        "devtools.netmonitor.har.forceExport");
+    }
 
     // Build HAR object.
     return this.buildHarData(options).then(har => {
       // Do not export an empty HAR file, unless the user
       // explicitly says so (using the forceExport option).
       if (!har.log.entries.length && !options.forceExport) {
         return Promise.resolve();
       }