Bug 1356543 - Copy image to clipboard via special MIME type
With this commit, web applications can put images on the clipboard by
assigning a Blob or File on a DataTransfer instance in the cut/copy
event, for the "application/x-moz-nativeimage" MIME-type.
MozReview-Commit-ID: LM4XBRhN5MS
--- a/dom/events/DataTransfer.cpp
+++ b/dom/events/DataTransfer.cpp
@@ -4,16 +4,17 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/ArrayUtils.h"
#include "mozilla/BasicEvents.h"
#include "DataTransfer.h"
+#include "imgTools.h"
#include "nsIDOMDocument.h"
#include "nsISupportsPrimitives.h"
#include "nsIScriptSecurityManager.h"
#include "mozilla/dom/DOMStringList.h"
#include "nsArray.h"
#include "nsError.h"
#include "nsIDragService.h"
#include "nsIClipboard.h"
@@ -1120,16 +1121,29 @@ DataTransfer::GetTransferable(uint32_t a
added = true;
}
}
}
// Clear the stream so it doesn't get used again.
stream = nullptr;
+ } else if (type.EqualsASCII(kNativeImageMime)) {
+ // This is the second pass of the loop and the native image type is
+ // encountered. Try to convert it to a format that can be recognized by
+ // external image editing software.
+
+ // When an image is to be put on the clipboard, it is decoded and the
+ // first frame is extracted. Since the resulting data will certainly
+ // be different from the input, the data is only stored as an image
+ // for one special MIME type, "application/x-moz-nativeimage".
+ // (and not for other types such as "image/png").
+ if (TryTransferAsImage(type, variant, transferable)) {
+ added = true;
+ }
} else {
// This is the second pass of the loop and a known type is encountered.
// Add it as is.
if (!ConvertFromVariant(variant, getter_AddRefs(convertedData),
&lengthInBytes)) {
continue;
}
@@ -1236,16 +1250,62 @@ DataTransfer::ConvertFromVariant(nsIVari
strSupports.forget(aSupports);
// each character is two bytes
*aLength = str.Length() << 1;
return true;
}
+bool
+DataTransfer::TryTransferAsImage(const nsAString& aFormat,
+ nsIVariant* aVariant,
+ nsITransferable* aTransferable) const
+{
+ nsCOMPtr<nsISupports> data;
+ nsresult rv = aVariant->GetAsISupports(getter_AddRefs(data));
+ NS_ENSURE_SUCCESS(rv, false);
+
+ nsCOMPtr<BlobImpl> blobImpl = do_QueryInterface(data);
+ if (!blobImpl) {
+ nsCOMPtr<nsIDOMBlob> domBlob = do_QueryInterface(data);
+ NS_ENSURE_TRUE(domBlob, false);
+ blobImpl = static_cast<Blob*>(domBlob.get())->Impl();
+ }
+
+ nsCOMPtr<nsIInputStream> imageStream;
+ IgnoredErrorResult error;
+ blobImpl->GetInternalStream(getter_AddRefs(imageStream), error);
+ if (NS_WARN_IF(error.Failed())) {
+ return false;
+ }
+
+ nsAutoString mimeTypeUTF16;
+ blobImpl->GetType(mimeTypeUTF16);
+ NS_ConvertUTF16toUTF8 mimeTypeUTF8(mimeTypeUTF16);
+
+ nsCOMPtr<imgITools> imgtool = do_GetService(NS_IMGTOOLS_CID);
+ NS_ENSURE_TRUE(imgtool, false);
+
+ nsCOMPtr<imgIContainer> imgContainer;
+ rv = imgtool->DecodeImage(imageStream, mimeTypeUTF8, getter_AddRefs(imgContainer));
+ NS_ENSURE_SUCCESS(rv, false);
+
+ nsCOMPtr<nsISupportsInterfacePointer> imgPtr =
+ do_CreateInstance(NS_SUPPORTS_INTERFACE_POINTER_CONTRACTID);
+ NS_ENSURE_TRUE(imgPtr, false);
+
+ rv = imgPtr->SetData(imgContainer);
+ NS_ENSURE_SUCCESS(rv, false);
+
+ rv = aTransferable->SetTransferData(kNativeImageMime, imgPtr, sizeof(nsISupports*));
+ NS_ENSURE_SUCCESS(rv, false);
+ return true;
+}
+
void
DataTransfer::ClearAll()
{
mItems->ClearAllItems();
}
uint32_t
DataTransfer::MozItemCount() const
--- a/dom/events/DataTransfer.h
+++ b/dom/events/DataTransfer.h
@@ -246,16 +246,22 @@ public:
GetTransferable(uint32_t aIndex, nsILoadContext* aLoadContext);
// converts the data in the variant to an nsISupportString if possible or
// an nsISupports or null otherwise.
bool ConvertFromVariant(nsIVariant* aVariant,
nsISupports** aSupports,
uint32_t* aLength) const;
+ // converts the data in the variant to an imgIContainer and store it in the
+ // nsITransferable object.
+ bool TryTransferAsImage(const nsAString& aFormat,
+ nsIVariant* aVariant,
+ nsITransferable* aTransferable) const;
+
// clears all of the data
void ClearAll();
// Similar to SetData except also specifies the principal to store.
// aData may be null when called from CacheExternalDragFormats or
// CacheExternalClipboardFormats.
nsresult SetDataWithPrincipal(const nsAString& aFormat,
nsIVariant* aData,
--- a/dom/events/moz.build
+++ b/dom/events/moz.build
@@ -140,16 +140,17 @@ LOCAL_INCLUDES += [
'/docshell/base',
'/dom/base',
'/dom/html',
'/dom/storage',
'/dom/svg',
'/dom/workers',
'/dom/xml',
'/dom/xul',
+ '/image',
'/js/xpconnect/wrappers',
'/layout/generic',
'/layout/xul',
'/layout/xul/tree/',
]
if CONFIG['GNU_CXX']:
CXXFLAGS += ['-Wno-error=shadow']
--- a/dom/events/test/mochitest.ini
+++ b/dom/events/test/mochitest.ini
@@ -164,16 +164,17 @@ skip-if = toolkit == 'android' #CRASH_DU
[test_legacy_event.html]
[test_messageEvent.html]
[test_messageEvent_init.html]
[test_moz_mouse_pixel_scroll_event.html]
[test_offsetxy.html]
[test_onerror_handler_args.html]
[test_passive_listeners.html]
[test_paste_image.html]
+[test_bug1356543.html]
[test_wheel_default_action.html]
[test_bug687787.html]
[test_bug1305458.html]
[test_bug1298970.html]
[test_bug1304044.html]
[test_bug1332699.html]
[test_bug1339758.html]
[test_dnd_with_modifiers.html]
new file mode 100644
--- /dev/null
+++ b/dom/events/test/test_bug1356543.html
@@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Test for bug 1356543</title>
+<link rel="stylesheet" href="/tests/SimpleTest/test.css">
+<script src="/tests/SimpleTest/SimpleTest.js"></script>
+<script src="/tests/SimpleTest/EventUtils.js"></script>
+
+<body>
+<div contentEditable id="editable"></div>
+<script class="testbody">
+
+const kNativeImageMime = "application/x-moz-nativeimage";
+const DUMMY_TEXT = "dummy text for clipboard";
+
+async function doCopyAsImage(data) {
+ // Clear the clipboard by unconditionally writing dummy data.
+ await new Promise(resolve => {
+ document.addEventListener("copy", function(event) {
+ event.preventDefault();
+ event.clipboardData.clearData();
+ event.clipboardData.setData('Text', DUMMY_TEXT);
+ resolve();
+ }, {once: true});
+
+ synthesizeKey("c", { accelKey: true });
+ });
+
+ // Pastes the actual data.
+ await new Promise(resolve => {
+ document.addEventListener("copy", function(event) {
+ event.preventDefault();
+ event.clipboardData.clearData();
+ event.clipboardData.mozSetDataAt(kNativeImageMime, data, 0);
+ resolve();
+ }, {once: true});
+
+ synthesizeKey("c", { accelKey: true });
+ });
+}
+
+async function doPaste(onPaste) {
+ await new Promise(resolve => {
+ document.addEventListener("paste", function(event) {
+ onPaste(event);
+ resolve();
+ }, {once: true});
+ editable.textContent = "";
+ editable.focus();
+ synthesizeKey("v", { accelKey: true });
+ });
+}
+
+async function pasteAndVerify(expectedMime, expectedWidth, expectedHeight) {
+ var file;
+ await doPaste((event) => {
+ is(event.clipboardData.types.length, 1, "expected one type");
+ is(event.clipboardData.types[0], "Files", "expected type");
+ is(event.clipboardData.files.length, 1, "expected one file");
+
+ var rawData = event.clipboardData.mozGetDataAt(kNativeImageMime, 0);
+ is(rawData, null, "expected no data for native image MIME");
+
+ file = event.clipboardData.files[0];
+ });
+
+ is(file.type, expectedMime, "expected file.type");
+
+ // event.files[0] should be an accurate representation of the input image.
+ var img = await loadBlobAsImage(file);
+ await verifyImage(img, expectedWidth, expectedHeight);
+
+ // This is the most significant part of the whole test file;
+ // this confirms that an image was put on the clipboard.
+ img = editable.querySelector("img");
+ ok(img, "should have pasted an image");
+ await verifyImage(img, expectedWidth, expectedHeight);
+}
+
+async function pasteAndExpectEmptyClipboard() {
+ await doPaste((event) => {
+ // Ideally all counters would be zero, but since there is no portable way to
+ // clear the clipboard, the doCopyAsImage function just writes a dummy value
+ // before copying the image.
+ is(event.clipboardData.types.length, 1, "expected one type");
+ is(event.clipboardData.types[0], "text/plain", "expected dummy type");
+ is(event.clipboardData.files.length, 0, "expected no file");
+ is(event.clipboardData.mozItemCount, 1, "expected clipboard to be empty");
+ is(event.clipboardData.getData("Text"), DUMMY_TEXT,
+ "expected clipboard to contain a dummy value");
+ });
+ var img = editable.querySelector("img");
+ is(img, null, "should not have pasted an image");
+}
+
+async function verifyImage(img, expectedWidth, expectedHeight) {
+ is(img.naturalWidth, expectedWidth, "image width should match");
+ is(img.naturalHeight, expectedHeight, "image height should match");
+
+ var canvas = document.createElement("canvas");
+ canvas.width = 1;
+ canvas.height = 1;
+ var ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0); // Draw without scaling.
+ var pixelData = ctx.getImageData(0, 0, 1, 1).data;
+ // expected 255, but jpeg has 254.
+ ok(pixelData[0] >= 254, `pixel ${pixelData[0]} should be red`);
+ // expected 0, but can be 38 after conversion to PNG.
+ ok(pixelData[1] <= 38, `pixel ${pixelData[1]} should not be green`);
+ is(pixelData[2], 0, `pixel ${pixelData[2]} should not be blue`);
+ is(pixelData[3], 255, `pixel ${pixelData[3]} should be opaque`);
+}
+
+async function loadBlobAsImage(blob) {
+ var img = new Image();
+ await new Promise((resolve, reject) => {
+ img.onload = resolve;
+ img.onerror = () => reject(new Error("Failed to load image: " + img.src));
+ img.src = URL.createObjectURL(blob);
+ });
+ return img;
+}
+
+function createBlob(type, b64data) {
+ var data = Uint8Array.from(atob(b64data), c => c.charCodeAt(0));
+ return new Blob([data], {type});
+}
+
+const JPEG_DATA64 = "/9j/4AAQSkZJRgABAQEAYABgAAD//gA+Q1JFQVRPUjogZ2QtanBlZyB2MS4wICh1c2luZyBJSkcgSlBFRyB2ODApLCBkZWZhdWx0IHF1YWxpdHkK/9sAQwAIBgYHBgUIBwcHCQkICgwUDQwLCwwZEhMPFB0aHx4dGhwcICQuJyAiLCMcHCg3KSwwMTQ0NB8nOT04MjwuMzQy/9sAQwEJCQkMCwwYDQ0YMiEcITIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy/8AAEQgAAQABAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A4uiiivmT9xP/2Q==";
+
+async function doTest() {
+ SimpleTest.waitForExplicitFinish();
+
+ // Red 1x1 images.
+ var testImages = [
+ createBlob("image/gif","R0lGODdhAQABAIAAAP8AAP///ywAAAAAAQABAAACAkQBADs="),
+ createBlob("image/jpeg", JPEG_DATA64),
+ createBlob("image/jpg", JPEG_DATA64),
+ createBlob("image/png", "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAABlBMVEX/AAD///9BHTQRAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAACklEQVQImWNgAAAAAgAB9HFkpgAAAABJRU5ErkJggg=="),
+ ];
+ var height = 1;
+ var width = 1;
+ for (var blob of testImages) {
+ info("Testing copy and paste of image of type " + blob.type);
+
+ // Sanity check: Verify that the input image is already valid.
+ verifyImage(await loadBlobAsImage(blob), width, height);
+
+ await doCopyAsImage(blob);
+ // When image data is copied, its first frame is decoded and exported to the
+ // clipboard. The pasted result is always an unanimated PNG file, regardless
+ // of the input.
+ await pasteAndVerify("image/png", width, height);
+ }
+
+ info("blobs without MIME type cannot be exported as an image");
+ await doCopyAsImage(new Blob([testImages[0]]));
+ await pasteAndExpectEmptyClipboard();
+
+ info("blos with invalid image data cannot be exported as an image");
+ await doCopyAsImage(new Blob([], { type: "image/png" }));
+ await pasteAndExpectEmptyClipboard();
+
+ info("non-Blob values cannot be exported as an image");
+ await doCopyAsImage("this is not a blob");
+ await pasteAndExpectEmptyClipboard();
+ await doCopyAsImage(null);
+ await pasteAndExpectEmptyClipboard();
+
+ SimpleTest.finish();
+}
+
+</script>
+<body onload="doTest();"></body>
+</html>