Bug 1356543 - Add clipboard.setImageData API draft
authorRob Wu <rob@robwu.nl>
Mon, 04 Sep 2017 21:43:06 +0200
changeset 659388 6390c43a2a31c2f3b35ab93abdc69c3a78dbe1e9
parent 659233 973e8b890a62aee4b3170558ac3b608928162ef6
child 729976 ca76609a37caac7510faccf40e19c448ad926e61
push id78115
push userbmo:rob@robwu.nl
push dateTue, 05 Sep 2017 23:00:35 +0000
bugs1356543
milestone57.0a1
Bug 1356543 - Add clipboard.setImageData API This introduces an implementation of the clipboard.setImageData API. I did not find any complete documentation about how copying and pasting images is supposed to work in Firefox, so I added many lines of documentation based on experimenting and reading the source code. The implementation is very similar to the Add-on SDK's implementation, save for one difference: The third parameter to setTransferData is 0 instead of -1. Its significance is elaborated in ext-clipboard.js. The newly added tests serve the following purposes: - Verification that clipboard.setImageData is working as expected. There is no way to test that pasting in an external application really works, so we just check whether Firefox recognizes the special image data by pasting in a contentEditable area. - Test coverage for reading clipboard data via the "paste" event and using event.clipboardData to access the pasted data, because this is the only way to read non-text data in a WebExtension extension. MozReview-Commit-ID: Ldrx7LCIta2
toolkit/components/extensions/ext-clipboard.js
toolkit/components/extensions/ext-toolkit.json
toolkit/components/extensions/jar.mn
toolkit/components/extensions/schemas/clipboard.json
toolkit/components/extensions/schemas/jar.mn
toolkit/components/extensions/test/mochitest/mochitest-common.ini
toolkit/components/extensions/test/mochitest/test_ext_clipboard_image.html
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/ext-clipboard.js
@@ -0,0 +1,84 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+"use strict";
+
+XPCOMUtils.defineLazyServiceGetter(this, "imgTools",
+                                   "@mozilla.org/image/tools;1", "imgITools");
+
+const ArrayBufferInputStream = Components.Constructor(
+    "@mozilla.org/io/arraybuffer-input-stream;1", "nsIArrayBufferInputStream");
+const SupportsInterfacePointer = Components.Constructor(
+    "@mozilla.org/supports-interface-pointer;1", "nsISupportsInterfacePointer");
+const Transferable = Components.Constructor(
+    "@mozilla.org/widget/transferable;1", "nsITransferable");
+
+this.clipboard = class extends ExtensionAPI {
+  getAPI(context) {
+    return {
+      clipboard: {
+        async setImageData(imageData, imageType) {
+          if (AppConstants.platform == "android") {
+            return Promise.reject({message: "Writing images to the clipboard is not supported on Android"});
+          }
+          let mimeType = `image/${imageType}`;
+          let input = new ArrayBufferInputStream();
+          input.setData(imageData, 0, imageData.byteLength);
+
+          let container;
+          try {
+            container = imgTools.decodeImage(input, mimeType);
+          } catch (e) {
+            return Promise.reject({message: `Data is not a valid ${imageType} image`});
+          }
+
+          // Other applications can only access the copied image once the data
+          // is exported via the platform-specific clipboard APIs:
+          // nsClipboard::SelectionGetEvent (widget/gtk/nsClipboard.cpp)
+          // nsClipboard::PasteDictFromTransferable (widget/cocoa/nsClipboard.mm)
+          // nsDataObj::GetDib (widget/windows/nsDataObj.cpp)
+          //
+          // The common protocol for exporting a nsITransferable as an image is:
+          // - Use nsITransferable::GetTransferData to fetch the stored data.
+          // - QI a nsISupportsInterfacePointer and get the underlying pointer.
+          // - QI imgIContainer on the pointer.
+          // - Convert the image to the native clipboard format.
+          //
+          // Below we create a nsITransferable in the above format.
+          let imgPtr = new SupportsInterfacePointer();
+          imgPtr.data = container;
+          let transferable = new Transferable();
+          transferable.init(null);
+          transferable.addDataFlavor(mimeType);
+
+          // Internal consumers expect the image data to be stored as a
+          // nsIInputStream. On Linux and Windows, pasted data is directly
+          // retrieved from the system's native clipboard, and made available
+          // as a nsIInputStream.
+          //
+          // On macOS, nsClipboard::GetNativeClipboardData (nsClipboard.mm) uses
+          // a cached copy of nsITransferable if available, e.g. when the copy
+          // was initiated by the same browser instance. Consequently, the
+          // transferable still holds a nsISupportsInterfacePointer pointer
+          // instead of a nsIInputStream, and logic that assumes the data to be
+          // a nsIInputStream instance fails.
+          // For example HTMLEditor::InsertObject (HTMLEditorDataTransfer.cpp)
+          // and DataTransferItem::FillInExternalData (DataTransferItem.cpp).
+          //
+          // As a work-around, we force nsClipboard::GetNativeClipboardData to
+          // ignore the cached image data, by passing zero as the length
+          // parameter to transferable.setTransferData. When the length is zero,
+          // nsITransferable::GetTransferData will return NS_ERROR_FAILURE and
+          // conveniently nsClipboard::GetNativeClipboardData will then fall
+          // back to retrieving the data directly from the system's clipboard.
+          //
+          // Note that the length itself is not really used if the data is not
+          // a string type, so the actual value does not matter.
+          transferable.setTransferData(mimeType, imgPtr, 0);
+
+          Services.clipboard.setData(
+            transferable, null, Services.clipboard.kGlobalClipboard);
+        },
+      },
+    };
+  }
+};
--- a/toolkit/components/extensions/ext-toolkit.json
+++ b/toolkit/components/extensions/ext-toolkit.json
@@ -19,16 +19,24 @@
   "browserSettings": {
     "url": "chrome://extensions/content/ext-browserSettings.js",
     "schema": "chrome://extensions/content/schemas/browser_settings.json",
     "scopes": ["addon_parent"],
     "paths": [
       ["browserSettings"]
     ]
   },
+  "clipboard": {
+    "url": "chrome://extensions/content/ext-clipboard.js",
+    "schema": "chrome://extensions/content/schemas/clipboard.json",
+    "scopes": ["addon_parent"],
+    "paths": [
+      ["clipboard"]
+    ]
+  },
   "contextualIdentities": {
     "url": "chrome://extensions/content/ext-contextualIdentities.js",
     "schema": "chrome://extensions/content/schemas/contextual_identities.json",
     "scopes": ["addon_parent"],
     "events": ["startup"],
     "permissions": ["contextualIdentities"],
     "paths": [
       ["contextualIdentities"]
--- a/toolkit/components/extensions/jar.mn
+++ b/toolkit/components/extensions/jar.mn
@@ -5,16 +5,17 @@
 toolkit.jar:
 % content extensions %content/extensions/
     content/extensions/dummy.xul
     content/extensions/ext-alarms.js
     content/extensions/ext-backgroundPage.js
     content/extensions/ext-browser-content.js
     content/extensions/ext-browserSettings.js
     content/extensions/ext-contextualIdentities.js
+    content/extensions/ext-clipboard.js
     content/extensions/ext-cookies.js
     content/extensions/ext-downloads.js
     content/extensions/ext-extension.js
     content/extensions/ext-i18n.js
     content/extensions/ext-idle.js
     content/extensions/ext-management.js
     content/extensions/ext-notifications.js
     content/extensions/ext-permissions.js
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/schemas/clipboard.json
@@ -0,0 +1,30 @@
+[
+  {
+    "namespace": "clipboard",
+    "description": "Offers the ability to write to the clipboard. Reading is not supported because the clipboard can already be read through the standard web platform APIs.",
+    "permissions": ["clipboardWrite"],
+    "functions": [
+      {
+        "name": "setImageData",
+        "type": "function",
+        "description": "Copy an image to the clipboard. The image is re-encoded before it is written to the clipboard. If the image is invalid, the clipboard is not modified.",
+        "async": true,
+        "parameters": [
+          {
+            "type": "object",
+            "isInstanceOf": "ArrayBuffer",
+            "additionalProperties": true,
+            "name": "imageData",
+            "description": "The image data to be copied."
+          },
+          {
+            "type": "string",
+            "name": "imageType",
+            "enum": ["jpeg", "png"],
+            "description": "The type of imageData."
+          }
+        ]
+      }
+    ]
+  }
+]
--- a/toolkit/components/extensions/schemas/jar.mn
+++ b/toolkit/components/extensions/schemas/jar.mn
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 toolkit.jar:
 % content extensions %content/extensions/
     content/extensions/schemas/alarms.json
     content/extensions/schemas/browser_settings.json
+    content/extensions/schemas/clipboard.json
     content/extensions/schemas/contextual_identities.json
     content/extensions/schemas/cookies.json
     content/extensions/schemas/downloads.json
     content/extensions/schemas/events.json
     content/extensions/schemas/experiments.json
     content/extensions/schemas/extension.json
     content/extensions/schemas/extension_types.json
     content/extensions/schemas/extension_protocol_handlers.json
--- a/toolkit/components/extensions/test/mochitest/mochitest-common.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
@@ -53,16 +53,17 @@ support-files =
   lorem.html.gz^headers^
   return_headers.sjs
   slow_response.sjs
   webrequest_worker.js
   !/toolkit/components/passwordmgr/test/authenticate.sjs
   !/dom/tests/mochitest/geolocation/network_geolocation.sjs
 
 [test_ext_clipboard.html]
+[test_ext_clipboard_image.html]
 # skip-if = # disabled test case with_permission_allow_copy, see inline comment.
 [test_ext_inIncognitoContext_window.html]
 skip-if = os == 'android' # Android does not support multiple windows.
 [test_ext_geturl.html]
 [test_ext_background_canvas.html]
 [test_ext_content_security_policy.html]
 [test_ext_contentscript_api_injection.html]
 [test_ext_contentscript_async_loading.html]
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/test/mochitest/test_ext_clipboard_image.html
@@ -0,0 +1,263 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Clipboard permissions tests</title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script src="/tests/SimpleTest/SpawnTask.js"></script>
+  <script src="/tests/SimpleTest/ExtensionTestUtils.js"></script>
+  <script src="head.js"></script>
+  <link rel="stylesheet" href="/tests/SimpleTest/test.css">
+</head>
+<body>
+
+<script>
+"use strict";
+/**
+ * This cannot be a xpcshell test, because:
+ * - On Android, copyString of nsIClipboardHelper segfaults because
+ *   widget/android/nsClipboard.cpp calls java::Clipboard::SetText, which is
+ *   unavailable in xpcshell.
+ * - On Windows, the clipboard is unavailable to xpcshell.
+ */
+
+function resetClipboard() {
+  SpecialPowers.clipboardCopyString(
+    "This is the default value of the clipboard in the test.");
+}
+
+async function checkClipboardHasTestImage(imageType) {
+  async function backgroundScript(imageType) {
+    async function verifyImage(img) {
+      // Checks whether the image is a 1x1 red image.
+      browser.test.assertEq(1, img.naturalWidth, "image width should match");
+      browser.test.assertEq(1, img.naturalHeight, "image height should match");
+
+      let canvas = document.createElement("canvas");
+      canvas.width = 1;
+      canvas.height = 1;
+      let ctx = canvas.getContext("2d");
+      ctx.drawImage(img, 0, 0);  // Draw without scaling.
+      let [r, g, b, a] = ctx.getImageData(0, 0, 1, 1).data;
+      let expectedColor;
+      if (imageType === "png") {
+        expectedColor = [255, 0, 0];
+      } else if (imageType === "jpeg") {
+        expectedColor = [254, 0, 0];
+      }
+      let {os} = await browser.runtime.getPlatformInfo();
+      if (os === "mac") {
+        // Due to https://bugzil.la/1396587, the pasted image differs from the
+        // original/expected image.
+        // Once that bug is fixed, this whole macOS-only branch can be removed.
+        if (imageType === "png") {
+          expectedColor = [255, 38, 0];
+        } else if (imageType === "jpeg") {
+          expectedColor = [255, 38, 0];
+        }
+      }
+      browser.test.assertEq(expectedColor[0], r, "pixel should be red");
+      browser.test.assertEq(expectedColor[1], g, "pixel should not contain green");
+      browser.test.assertEq(expectedColor[2], b, "pixel should not contain blue");
+      browser.test.assertEq(255, a, "pixel should be opaque");
+    }
+
+    let editable = document.body;
+    editable.contentEditable = true;
+    let file;
+    await new Promise(resolve => {
+      document.addEventListener("paste", function(event) {
+        browser.test.assertEq(1, event.clipboardData.types.length, "expected one type");
+        browser.test.assertEq("Files", event.clipboardData.types[0], "expected type");
+        browser.test.assertEq(1, event.clipboardData.files.length, "expected one file");
+
+        // After returning from the paste event, event.clipboardData is cleaned, so we
+        // have to store the file in a separate variable.
+        file = event.clipboardData.files[0];
+        resolve();
+      }, {once: true});
+
+      document.execCommand("paste");  // requires clipboardWrite permission.
+    });
+
+    // When image data is copied, its first frame is decoded and exported to the
+    // clipboard. The pasted result is always an unanimated PNG file, regardless
+    // of the input.
+    browser.test.assertEq("image/png", file.type, "expected file.type");
+
+    // event.files[0] should be an accurate representation of the input image.
+    {
+      let img = new Image();
+      await new Promise((resolve, reject) => {
+        img.onload = resolve;
+        img.onerror = () => reject(new Error(`Failed to load image ${img.src} of size ${file.size}`));
+        img.src = URL.createObjectURL(file);
+      });
+
+      await verifyImage(img);
+    }
+
+    // This confirms that an image was put on the clipboard.
+    // In contrast, when document.execCommand('copy') + clipboardData.setData
+    // is used, then the 'paste' event will also have the image data (as tested
+    // above), but the contentEditable area will be empty.
+    {
+      let imgs = editable.querySelectorAll("img");
+      browser.test.assertEq(1, imgs.length, "should have pasted one image");
+      await verifyImage(imgs[0]);
+    }
+    browser.test.sendMessage("tested image on clipboard");
+  }
+
+  let extension = ExtensionTestUtils.loadExtension({
+    background: `(${backgroundScript})("${imageType}");`,
+    manifest: {
+      permissions: ["clipboardRead"],
+    },
+  });
+  await extension.startup();
+  await extension.awaitMessage("tested image on clipboard");
+  await extension.unload();
+}
+
+add_task(async function test_without_clipboard_permission() {
+  let extension = ExtensionTestUtils.loadExtension({
+    background() {
+      browser.test.assertEq(undefined, browser.clipboard,
+        "clipboard API requires the clipboardWrite permission.");
+      browser.test.notifyPass();
+    },
+    manifest: {
+      permissions: ["clipboardRead"],
+    },
+  });
+  await extension.startup();
+  await extension.awaitFinish();
+  await extension.unload();
+});
+
+add_task(async function test_copy_png() {
+  if (AppConstants.platform === "android") {
+    return;  // Android does not support images on the clipboard.
+  }
+  let extension = ExtensionTestUtils.loadExtension({
+    async background() {
+      // A 1x1 red PNG image.
+      let b64data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAABlBMVEX/AAD///9BHTQRAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAACklEQVQImWNgAAAAAgAB9HFkpgAAAABJRU5ErkJggg==";
+      let imageData = Uint8Array.from(atob(b64data), c => c.charCodeAt(0)).buffer;
+      await browser.clipboard.setImageData(imageData, "png");
+      browser.test.sendMessage("Called setImageData with PNG");
+    },
+    manifest: {
+      permissions: ["clipboardWrite"],
+    },
+  });
+
+  resetClipboard();
+
+  await extension.startup();
+  await extension.awaitMessage("Called setImageData with PNG");
+  await extension.unload();
+
+  await checkClipboardHasTestImage("png");
+});
+
+add_task(async function test_copy_jpeg() {
+  if (AppConstants.platform === "android") {
+    return;  // Android does not support images on the clipboard.
+  }
+  let extension = ExtensionTestUtils.loadExtension({
+    async background() {
+      // A 1x1 red JPEG image, created using: convert xc:red red.jpg.
+      // JPEG is lossy, and the red pixel value is actually #FE0000 instead of
+      // #FF0000 (also seen using: convert red.jpg text:-).
+      let b64data = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGCQgKCgkICQkKDA8MCgsOCwkJDRENDg8QEBEQCgwSExIQEw8QEBD/2wBDAQMDAwQDBAgEBAgQCwkLEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBD/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACP/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAVAQEBAAAAAAAAAAAAAAAAAAAHCf/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/ADoDFU3/2Q==";
+      let imageData = Uint8Array.from(atob(b64data), c => c.charCodeAt(0)).buffer;
+      await browser.clipboard.setImageData(imageData, "jpeg");
+      browser.test.sendMessage("Called setImageData with JPEG");
+    },
+    manifest: {
+      permissions: ["clipboardWrite"],
+    },
+  });
+
+  resetClipboard();
+
+  await extension.startup();
+  await extension.awaitMessage("Called setImageData with JPEG");
+  await extension.unload();
+
+  await checkClipboardHasTestImage("jpeg");
+});
+
+add_task(async function test_copy_invalid_image() {
+  if (AppConstants.platform === "android") {
+    // Android does not support images on the clipboard.
+    return;
+  }
+  let extension = ExtensionTestUtils.loadExtension({
+    async background() {
+      // This is a PNG image.
+      let b64data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAABlBMVEX/AAD///9BHTQRAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAACklEQVQImWNgAAAAAgAB9HFkpgAAAABJRU5ErkJggg==";
+      let pngImageData = Uint8Array.from(atob(b64data), c => c.charCodeAt(0)).buffer;
+      await browser.test.assertRejects(
+        browser.clipboard.setImageData(pngImageData, "jpeg"),
+        "Data is not a valid jpeg image",
+        "Image data that is not valid for the given type should be rejected.");
+      browser.test.sendMessage("finished invalid image");
+    },
+    manifest: {
+      permissions: ["clipboardWrite"],
+    },
+  });
+
+  await extension.startup();
+  await extension.awaitMessage("finished invalid image");
+  await extension.unload();
+});
+
+add_task(async function test_copy_invalid_image_type() {
+  let extension = ExtensionTestUtils.loadExtension({
+    async background() {
+      // setImageData expects "png" or "jpeg", but we pass "image/png" here.
+      browser.test.assertThrows(
+        () => { browser.clipboard.setImageData(new ArrayBuffer(0), "image/png"); },
+        "Type error for parameter imageType (Invalid enumeration value \"image/png\") for clipboard.setImageData.",
+        "An invalid type for setImageData should be rejected.");
+      browser.test.sendMessage("finished invalid type");
+    },
+    manifest: {
+      permissions: ["clipboardWrite"],
+    },
+  });
+
+  await extension.startup();
+  await extension.awaitMessage("finished invalid type");
+  await extension.unload();
+});
+
+if (AppConstants.platform === "android") {
+  add_task(async function test_setImageData_unsupported_on_android() {
+    let extension = ExtensionTestUtils.loadExtension({
+      async background() {
+        // Android does not support images on the clipboard,
+        // so it should not try to decode an image but fail immediately.
+        await browser.test.assertRejects(
+          browser.clipboard.setImageData(new ArrayBuffer(0), "png"),
+          "Writing images to the clipboard is not supported on Android",
+          "Should get an error when setImageData is called on Android.");
+        browser.test.sendMessage("finished unsupported setImageData");
+      },
+      manifest: {
+        permissions: ["clipboardWrite"],
+      },
+    });
+
+    await extension.startup();
+    await extension.awaitMessage("finished unsupported setImageData");
+    await extension.unload();
+  });
+}
+
+</script>
+</body>
+</html>