Bug 1356543 - Copy image to clipboard via special MIME type draft
authorRob Wu <rob@robwu.nl>
Sun, 02 Jul 2017 02:11:40 -0700
changeset 604148 4a9a4aea04579ead8133774ef6c18575fb053f74
parent 599733 594cc32b632396a867ef1f98428968b224d82151
child 604149 d0359806a78e3a144d10bf040ecf5a35784e6ee6
push id66977
push userbmo:rob@robwu.nl
push dateWed, 05 Jul 2017 12:39:50 +0000
bugs1356543
milestone56.0a1
Bug 1356543 - Copy image to clipboard via special MIME type With this commit, web applications can put images on the clipboard by assigning a Blob or File on a DataTransfer instance in the cut/copy event, for the "application/x-moz-nativeimage" MIME-type. MozReview-Commit-ID: LM4XBRhN5MS
dom/events/DataTransfer.cpp
dom/events/DataTransfer.h
dom/events/moz.build
dom/events/test/mochitest.ini
dom/events/test/test_bug1356543.html
--- a/dom/events/DataTransfer.cpp
+++ b/dom/events/DataTransfer.cpp
@@ -4,16 +4,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/BasicEvents.h"
 
 #include "DataTransfer.h"
 
+#include "imgTools.h"
 #include "nsIDOMDocument.h"
 #include "nsISupportsPrimitives.h"
 #include "nsIScriptSecurityManager.h"
 #include "mozilla/dom/DOMStringList.h"
 #include "nsArray.h"
 #include "nsError.h"
 #include "nsIDragService.h"
 #include "nsIClipboard.h"
@@ -1120,16 +1121,29 @@ DataTransfer::GetTransferable(uint32_t a
 
               added = true;
             }
           }
         }
 
         // Clear the stream so it doesn't get used again.
         stream = nullptr;
+      } else if (type.EqualsASCII(kNativeImageMime)) {
+        // This is the second pass of the loop and the native image type is
+        // encountered. Try to convert it to a format that can be recognized by
+        // external image editing software.
+
+        // When an image is to be put on the clipboard, it is decoded and the
+        // first frame is extracted. Since the resulting data will certainly
+        // be different from the input, the data is only stored as an image
+        // for one special MIME type, "application/x-moz-nativeimage".
+        // (and not for other types such as "image/png").
+        if (TryTransferAsImage(type, variant, transferable)) {
+          added = true;
+        }
       } else {
         // This is the second pass of the loop and a known type is encountered.
         // Add it as is.
         if (!ConvertFromVariant(variant, getter_AddRefs(convertedData),
                                 &lengthInBytes)) {
           continue;
         }
 
@@ -1236,16 +1250,62 @@ DataTransfer::ConvertFromVariant(nsIVari
   strSupports.forget(aSupports);
 
   // each character is two bytes
   *aLength = str.Length() << 1;
 
   return true;
 }
 
+bool
+DataTransfer::TryTransferAsImage(const nsAString& aFormat,
+                                 nsIVariant* aVariant,
+                                 nsITransferable* aTransferable) const
+{
+  nsCOMPtr<nsISupports> data;
+  nsresult rv = aVariant->GetAsISupports(getter_AddRefs(data));
+  NS_ENSURE_SUCCESS(rv, false);
+
+  nsCOMPtr<BlobImpl> blobImpl = do_QueryInterface(data);
+  if (!blobImpl) {
+    nsCOMPtr<nsIDOMBlob> domBlob = do_QueryInterface(data);
+    NS_ENSURE_TRUE(domBlob, false);
+    blobImpl = static_cast<Blob*>(domBlob.get())->Impl();
+  }
+
+  nsCOMPtr<nsIInputStream> imageStream;
+  IgnoredErrorResult error;
+  blobImpl->GetInternalStream(getter_AddRefs(imageStream), error);
+  if (NS_WARN_IF(error.Failed())) {
+    return false;
+  }
+
+  nsAutoString mimeTypeUTF16;
+  blobImpl->GetType(mimeTypeUTF16);
+  NS_ConvertUTF16toUTF8 mimeTypeUTF8(mimeTypeUTF16);
+
+  nsCOMPtr<imgITools> imgtool = do_GetService(NS_IMGTOOLS_CID);
+  NS_ENSURE_TRUE(imgtool, false);
+
+  nsCOMPtr<imgIContainer> imgContainer;
+  rv = imgtool->DecodeImage(imageStream, mimeTypeUTF8, getter_AddRefs(imgContainer));
+  NS_ENSURE_SUCCESS(rv, false);
+
+  nsCOMPtr<nsISupportsInterfacePointer> imgPtr =
+    do_CreateInstance(NS_SUPPORTS_INTERFACE_POINTER_CONTRACTID);
+  NS_ENSURE_TRUE(imgPtr, false);
+
+  rv = imgPtr->SetData(imgContainer);
+  NS_ENSURE_SUCCESS(rv, false);
+
+  rv = aTransferable->SetTransferData(kNativeImageMime, imgPtr, sizeof(nsISupports*));
+  NS_ENSURE_SUCCESS(rv, false);
+  return true;
+}
+
 void
 DataTransfer::ClearAll()
 {
   mItems->ClearAllItems();
 }
 
 uint32_t
 DataTransfer::MozItemCount() const
--- a/dom/events/DataTransfer.h
+++ b/dom/events/DataTransfer.h
@@ -246,16 +246,22 @@ public:
   GetTransferable(uint32_t aIndex, nsILoadContext* aLoadContext);
 
   // converts the data in the variant to an nsISupportString if possible or
   // an nsISupports or null otherwise.
   bool ConvertFromVariant(nsIVariant* aVariant,
                           nsISupports** aSupports,
                           uint32_t* aLength) const;
 
+  // converts the data in the variant to an imgIContainer and store it in the
+  // nsITransferable object.
+  bool TryTransferAsImage(const nsAString& aFormat,
+                          nsIVariant* aVariant,
+                          nsITransferable* aTransferable) const;
+
   // clears all of the data
   void ClearAll();
 
   // Similar to SetData except also specifies the principal to store.
   // aData may be null when called from CacheExternalDragFormats or
   // CacheExternalClipboardFormats.
   nsresult SetDataWithPrincipal(const nsAString& aFormat,
                                 nsIVariant* aData,
--- a/dom/events/moz.build
+++ b/dom/events/moz.build
@@ -140,16 +140,17 @@ LOCAL_INCLUDES += [
     '/docshell/base',
     '/dom/base',
     '/dom/html',
     '/dom/storage',
     '/dom/svg',
     '/dom/workers',
     '/dom/xml',
     '/dom/xul',
+    '/image',
     '/js/xpconnect/wrappers',
     '/layout/generic',
     '/layout/xul',
     '/layout/xul/tree/',
 ]
 
 if CONFIG['GNU_CXX']:
     CXXFLAGS += ['-Wno-error=shadow']
--- a/dom/events/test/mochitest.ini
+++ b/dom/events/test/mochitest.ini
@@ -164,16 +164,17 @@ skip-if = toolkit == 'android' #CRASH_DU
 [test_legacy_event.html]
 [test_messageEvent.html]
 [test_messageEvent_init.html]
 [test_moz_mouse_pixel_scroll_event.html]
 [test_offsetxy.html]
 [test_onerror_handler_args.html]
 [test_passive_listeners.html]
 [test_paste_image.html]
+[test_bug1356543.html]
 [test_wheel_default_action.html]
 [test_bug687787.html]
 [test_bug1305458.html]
 [test_bug1298970.html]
 [test_bug1304044.html]
 [test_bug1332699.html]
 [test_bug1339758.html]
 [test_dnd_with_modifiers.html]
new file mode 100644
--- /dev/null
+++ b/dom/events/test/test_bug1356543.html
@@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Test for bug 1356543</title>
+<link rel="stylesheet" href="/tests/SimpleTest/test.css">
+<script src="/tests/SimpleTest/SimpleTest.js"></script>
+<script src="/tests/SimpleTest/EventUtils.js"></script>
+
+<body>
+<div contentEditable id="editable"></div>
+<script class="testbody">
+
+const kNativeImageMime = "application/x-moz-nativeimage";
+const DUMMY_TEXT = "dummy text for clipboard";
+
+async function doCopyAsImage(data) {
+  // Clear the clipboard by unconditionally writing dummy data.
+  await new Promise(resolve => {
+    document.addEventListener("copy", function(event) {
+      event.preventDefault();
+      event.clipboardData.clearData();
+      event.clipboardData.setData('Text', DUMMY_TEXT);
+      resolve();
+    }, {once: true});
+    
+    synthesizeKey("c", { accelKey: true });
+  });
+
+  // Pastes the actual data.
+  await new Promise(resolve => {
+    document.addEventListener("copy", function(event) {
+      event.preventDefault();
+      event.clipboardData.clearData();
+      event.clipboardData.mozSetDataAt(kNativeImageMime, data, 0);
+      resolve();
+    }, {once: true});
+    
+    synthesizeKey("c", { accelKey: true });
+  });
+}
+
+async function doPaste(onPaste) {
+  await new Promise(resolve => {
+    document.addEventListener("paste", function(event) {
+      onPaste(event);
+      resolve();
+    }, {once: true});
+    editable.textContent = "";
+    editable.focus();
+    synthesizeKey("v", { accelKey: true });
+  });
+}
+
+async function pasteAndVerify(expectedMime, expectedWidth, expectedHeight) {
+  var file;
+  await doPaste((event) => {
+    is(event.clipboardData.types.length, 1, "expected one type");
+    is(event.clipboardData.types[0], "Files", "expected type");
+    is(event.clipboardData.files.length, 1, "expected one file");
+
+    var rawData = event.clipboardData.mozGetDataAt(kNativeImageMime, 0);
+    is(rawData, null, "expected no data for native image MIME");
+
+    file = event.clipboardData.files[0];
+  });
+
+  is(file.type, expectedMime, "expected file.type");
+
+  // event.files[0] should be an accurate representation of the input image.
+  var img = await loadBlobAsImage(file);
+  await verifyImage(img, expectedWidth, expectedHeight);
+
+  // This is the most significant part of the whole test file;
+  // this confirms that an image was put on the clipboard.
+  img = editable.querySelector("img");
+  ok(img, "should have pasted an image");
+  await verifyImage(img, expectedWidth, expectedHeight);
+}
+
+async function pasteAndExpectEmptyClipboard() {
+  await doPaste((event) => {
+    // Ideally all counters would be zero, but since there is no portable way to
+    // clear the clipboard, the doCopyAsImage function just writes a dummy value
+    // before copying the image.
+    is(event.clipboardData.types.length, 1, "expected one type");
+    is(event.clipboardData.types[0], "text/plain", "expected dummy type");
+    is(event.clipboardData.files.length, 0, "expected no file");
+    is(event.clipboardData.mozItemCount, 1, "expected clipboard to be empty");
+    is(event.clipboardData.getData("Text"), DUMMY_TEXT,
+       "expected clipboard to contain a dummy value");
+  });
+  var img = editable.querySelector("img");
+  is(img, null, "should not have pasted an image");
+}
+
+async function verifyImage(img, expectedWidth, expectedHeight) {
+  is(img.naturalWidth, expectedWidth, "image width should match");
+  is(img.naturalHeight, expectedHeight, "image height should match");
+
+  var canvas = document.createElement("canvas");
+  canvas.width = 1;
+  canvas.height = 1;
+  var ctx = canvas.getContext("2d");
+  ctx.drawImage(img, 0, 0);  // Draw without scaling.
+  var pixelData = ctx.getImageData(0, 0, 1, 1).data;
+  // expected 255, but jpeg has 254.
+  ok(pixelData[0] >= 254, `pixel ${pixelData[0]} should be red`);
+  // expected 0, but can be 38 after conversion to PNG.
+  ok(pixelData[1] <= 38, `pixel ${pixelData[1]} should not be green`);
+  is(pixelData[2], 0, `pixel ${pixelData[2]} should not be blue`);
+  is(pixelData[3], 255, `pixel ${pixelData[3]} should be opaque`);
+}
+
+async function loadBlobAsImage(blob) {
+  var img = new Image();
+  await new Promise((resolve, reject) => {
+    img.onload = resolve;
+    img.onerror = () => reject(new Error("Failed to load image: " + img.src));
+    img.src = URL.createObjectURL(blob);
+  });
+  return img;
+}
+
+function createBlob(type, b64data) {
+  var data = Uint8Array.from(atob(b64data), c => c.charCodeAt(0));
+  return new Blob([data], {type});
+}
+
+const JPEG_DATA64 = "/9j/4AAQSkZJRgABAQEAYABgAAD//gA+Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcgSlBFRyB2ODApLCBkZWZhdWx0IHF1YWxpdHkK/9sAQwAIBgYHBgUIBwcHCQkICgwUDQwLCwwZEhMPFB0aHx4dGhwcICQuJyAiLCMcHCg3KSwwMTQ0NB8nOT04MjwuMzQy/9sAQwEJCQkMCwwYDQ0YMiEcITIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy/8AAEQgAAQABAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A4uiiivmT9xP/2Q==";
+
+async function doTest() {
+  SimpleTest.waitForExplicitFinish();
+
+  // Red 1x1 images.
+  var testImages = [
+    createBlob("image/gif","R0lGODdhAQABAIAAAP8AAP///ywAAAAAAQABAAACAkQBADs="),
+    createBlob("image/jpeg", JPEG_DATA64),
+    createBlob("image/jpg", JPEG_DATA64),
+    createBlob("image/png", "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAABlBMVEX/AAD///9BHTQRAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAACklEQVQImWNgAAAAAgAB9HFkpgAAAABJRU5ErkJggg=="),
+  ];
+  var height = 1;
+  var width = 1;
+  for (var blob of testImages) {
+    info("Testing copy and paste of image of type " + blob.type);
+
+    // Sanity check: Verify that the input image is already valid.
+    verifyImage(await loadBlobAsImage(blob), width, height);
+
+    await doCopyAsImage(blob);
+    // When image data is copied, its first frame is decoded and exported to the
+    // clipboard. The pasted result is always an unanimated PNG file, regardless
+    // of the input.
+    await pasteAndVerify("image/png", width, height);
+  }
+
+  info("blobs without MIME type cannot be exported as an image");
+  await doCopyAsImage(new Blob([testImages[0]]));
+  await pasteAndExpectEmptyClipboard();
+
+  info("blos with invalid image data cannot be exported as an image");
+  await doCopyAsImage(new Blob([], { type: "image/png" }));
+  await pasteAndExpectEmptyClipboard();
+
+  info("non-Blob values cannot be exported as an image");
+  await doCopyAsImage("this is not a blob");
+  await pasteAndExpectEmptyClipboard();
+  await doCopyAsImage(null);
+  await pasteAndExpectEmptyClipboard();
+
+  SimpleTest.finish();
+}
+
+</script>
+<body onload="doTest();"></body>
+</html>