Bug 1403542 - Adds support for images with headless clipboard. r?jrmuizel draft
authorBrendan Dahl <brendan.dahl@gmail.com>
Tue, 14 Nov 2017 17:34:28 -0800
changeset 698441 e49d8b77e505ef6a72fc2050fd84119a3dc35261
parent 697940 f0c0fb9182d695081edf170d8e3bcb8164f2c96a
child 740380 c729fa16bc7f1ac74eef0a889f0d4e4f9ef2f7cd
push id89290
push userbmo:bdahl@mozilla.com
push dateWed, 15 Nov 2017 19:36:13 +0000
reviewersjrmuizel
bugs1403542
milestone59.0a1
Bug 1403542 - Adds support for images with headless clipboard. r?jrmuizel Images can now be copied into the clipboard and pasted without conversion. Native images, PNGs, and JPEGs are also able to be converted to PNGs or JPEGS when pasted out of the clibpoard. MozReview-Commit-ID: EwSfeVwxes6
dom/base/test/mochitest.ini
dom/events/test/mochitest.ini
toolkit/components/extensions/test/mochitest/mochitest-common.ini
widget/headless/HeadlessClipboard.cpp
widget/headless/HeadlessClipboard.h
widget/headless/HeadlessClipboardData.cpp
widget/headless/HeadlessClipboardData.h
widget/headless/moz.build
--- a/dom/base/test/mochitest.ini
+++ b/dom/base/test/mochitest.ini
@@ -290,17 +290,17 @@ skip-if = toolkit == 'android'
 [test_bug311681.xml]
 [test_bug313646.html]
 [test_bug320799.html]
 [test_bug322317.html]
 [test_bug326337.html]
 [test_bug330925.xhtml]
 [test_bug331959.html]
 [test_bug333064.html]
-skip-if = toolkit == 'android' || headless # Headless: Bug 1405868
+skip-if = toolkit == 'android'
 [test_bug333198.html]
 [test_bug333673.html]
 [test_bug337631.html]
 [test_bug338541.xhtml]
 [test_bug338583.html]
 skip-if = toolkit == 'android'
 [test_bug338679.html]
 [test_bug339494.html]
--- a/dom/events/test/mochitest.ini
+++ b/dom/events/test/mochitest.ini
@@ -170,17 +170,16 @@ skip-if = toolkit == 'android' #CRASH_DU
 [test_legacy_event.html]
 [test_messageEvent.html]
 [test_messageEvent_init.html]
 [test_moz_mouse_pixel_scroll_event.html]
 [test_offsetxy.html]
 [test_onerror_handler_args.html]
 [test_passive_listeners.html]
 [test_paste_image.html]
-skip-if = headless # Bug 1405869
 [test_wheel_default_action.html]
 [test_bug687787.html]
 [test_bug1305458.html]
 [test_bug1298970.html]
 [test_bug1304044.html]
 [test_bug1332699.html]
 [test_bug1339758.html]
 [test_bug1369072.html]
--- a/toolkit/components/extensions/test/mochitest/mochitest-common.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
@@ -56,17 +56,17 @@ support-files =
   return_headers.sjs
   slow_response.sjs
   webrequest_worker.js
   !/toolkit/components/passwordmgr/test/authenticate.sjs
   !/dom/tests/mochitest/geolocation/network_geolocation.sjs
 
 [test_ext_clipboard.html]
 [test_ext_clipboard_image.html]
-skip-if = headless # disabled test case with_permission_allow_copy, see inline comment. Headless: Bug 1405872
+# skip-if = # disabled test case with_permission_allow_copy, see inline comment.
 [test_ext_inIncognitoContext_window.html]
 skip-if = os == 'android' # Android does not support multiple windows.
 [test_ext_geturl.html]
 [test_ext_background_canvas.html]
 [test_ext_content_security_policy.html]
 [test_ext_contentscript_api_injection.html]
 [test_ext_contentscript_async_loading.html]
 skip-if = os == 'android' && debug # The generated script takes too long to load on Android debug
--- a/widget/headless/HeadlessClipboard.cpp
+++ b/widget/headless/HeadlessClipboard.cpp
@@ -1,111 +1,241 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "HeadlessClipboard.h"
 
 #include "nsISupportsPrimitives.h"
 #include "nsComponentManagerUtils.h"
-#include "nsCOMPtr.h"
+#include "nsString.h"
+#include "mozilla/RefPtr.h"
+#include "nsArrayUtils.h"
+#include "imgIContainer.h"
+#include "gfxUtils.h"
+#include "nsIInputStream.h"
+
+using mozilla::gfx::SourceSurface;
+using mozilla::gfx::DataSourceSurface;
 
 namespace mozilla {
 namespace widget {
 
 NS_IMPL_ISUPPORTS(HeadlessClipboard, nsIClipboard)
 
 HeadlessClipboard::HeadlessClipboard()
-  : mClipboard(MakeUnique<HeadlessClipboardData>())
 {
 }
 
 NS_IMETHODIMP
 HeadlessClipboard::SetData(nsITransferable *aTransferable,
                      nsIClipboardOwner *anOwner,
                      int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
-  // Clear out the clipboard in order to set the new data.
-  EmptyClipboard(aWhichClipboard);
-
-  // Only support plain text for now.
-  nsCOMPtr<nsISupports> clip;
-  uint32_t len;
-  nsresult rv = aTransferable->GetTransferData(kUnicodeMime,
-                                               getter_AddRefs(clip),
-                                               &len);
-  if (NS_FAILED(rv)) {
-    return rv;
-  }
-  nsCOMPtr<nsISupportsString> wideString = do_QueryInterface(clip);
-  if (!wideString) {
-    return NS_ERROR_NOT_IMPLEMENTED;
-  }
-  nsAutoString utf16string;
-  wideString->GetData(utf16string);
-  mClipboard->SetText(utf16string);
+  mTransferable = aTransferable;
 
   return NS_OK;
 }
 
+// Tries to unwrap aItem and convert it to the requested aOutputFlavor mime
+// type and then set the transfer data on the transferable. Currently, only
+// images are supported.
+bool
+TrySetTransferData(nsISupports* aItem, const char *aOutputFlavor,
+                   nsITransferable *aTransferable)
+{
+  nsCOMPtr<nsISupportsInterfacePointer> ptrPrimitive(do_QueryInterface(aItem));
+  if (!ptrPrimitive) {
+    return false;
+  }
+
+  nsCOMPtr<nsISupports> primitiveData;
+  ptrPrimitive->GetData(getter_AddRefs(primitiveData));
+  nsCOMPtr<imgIContainer> image(do_QueryInterface(primitiveData));
+  if (!image) {
+    return false;
+  }
+
+  RefPtr<SourceSurface> surface =
+    image->GetFrame(imgIContainer::FRAME_CURRENT,
+                    imgIContainer::FLAG_SYNC_DECODE);
+  if (!surface) {
+    return false;
+  }
+  RefPtr<DataSourceSurface> dataSurface = surface->GetDataSurface();
+  nsCOMPtr<nsIInputStream> imgStream;
+  gfxUtils::GetInputStream(dataSurface,
+                           true,
+                           aOutputFlavor,
+                           EmptyString().get(),
+                           getter_AddRefs(imgStream));
+  nsresult rv = aTransferable->SetTransferData(aOutputFlavor,
+                                               imgStream,
+                                               sizeof(nsIInputStream*));
+  return NS_SUCCEEDED(rv);
+}
+
+bool
+HeadlessClipboard::GetSourceFlavor(const char* aRequestedFlavor,
+                                   nsACString &result) {
+  if (!mTransferable) {
+    result.Assign(EmptyCString());
+    return false;
+  }
+  nsCOMPtr<nsIArray> supportedFlavors;
+  nsresult rv =
+    mTransferable->FlavorsTransferableCanExport(getter_AddRefs(supportedFlavors));
+  NS_ENSURE_SUCCESS(rv, false);
+  uint32_t supportedCount;
+  supportedFlavors->GetLength(&supportedCount);
+
+  // First try to see if the requested flavor is supported by default.
+  for (uint32_t i = 0; i < supportedCount; i++) {
+    nsCOMPtr<nsISupportsCString> supportedFlavor =
+      do_QueryElementAt(supportedFlavors, i);
+    if (!supportedFlavor) {
+      continue;
+    }
+    nsAutoCString supportedflavorStr;
+    supportedFlavor->GetData(supportedflavorStr);
+    if (supportedflavorStr.Equals(aRequestedFlavor)) {
+      result.Assign(supportedflavorStr);
+      return true;
+    }
+  }
+
+  // Now try the image formats that are convertible.
+  if (strcmp(aRequestedFlavor, kPNGImageMime) != 0 &&
+      strcmp(aRequestedFlavor, kJPGImageMime) != 0 &&
+      strcmp(aRequestedFlavor, kJPEGImageMime) != 0) {
+    result.Assign(EmptyCString());
+    return false;
+  }
+
+  for (uint32_t i = 0; i < supportedCount; i++) {
+    nsCOMPtr<nsISupportsCString> supportedFlavor =
+      do_QueryElementAt(supportedFlavors, i);
+    if (!supportedFlavor) {
+      continue;
+    }
+    nsAutoCString supportedflavorStr;
+    supportedFlavor->GetData(supportedflavorStr);
+    if (supportedflavorStr.EqualsLiteral(kNativeImageMime) ||
+        supportedflavorStr.EqualsLiteral(kPNGImageMime) ||
+        supportedflavorStr.EqualsLiteral(kJPGImageMime) ||
+        supportedflavorStr.EqualsLiteral(kJPEGImageMime)) {
+      result.Assign(supportedflavorStr);
+      return true;
+    }
+  }
+  result.Assign(EmptyCString());
+  return false;
+}
+
 NS_IMETHODIMP
 HeadlessClipboard::GetData(nsITransferable *aTransferable,
-                     int32_t aWhichClipboard)
+                           int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
-  nsresult rv;
-  nsCOMPtr<nsISupportsString> dataWrapper =
-    do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
-  rv = dataWrapper->SetData(mClipboard->GetText());
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  nsCOMPtr<nsIArray> requestedFlavors;
+  nsresult rv =
+    aTransferable->FlavorsTransferableCanImport(getter_AddRefs(requestedFlavors));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  uint32_t requestedCount;
+  requestedFlavors->GetLength(&requestedCount);
+  // See if the stored transferable supports any of the requested flavors.
+  for (uint32_t i = 0; i < requestedCount; i++) {
+    nsCOMPtr<nsISupportsCString> requestedFlavor =
+      do_QueryElementAt(requestedFlavors, i);
+    if (!requestedFlavor) {
+      continue;
+    }
+    nsAutoCString requestedFlavorStr;
+    rv = requestedFlavor->GetData(requestedFlavorStr);
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    nsAutoCString sourceFlavorStr;
+    if (!GetSourceFlavor(requestedFlavorStr.get(), sourceFlavorStr)) {
+      continue;
+    }
+
+    nsCOMPtr<nsISupports> item;
+    uint32_t len;
+
+    // First, check if the saved transferable supports the requested flavor with
+    // no conversion.
+    if (requestedFlavorStr == sourceFlavorStr) {
+      rv = mTransferable->GetTransferData(requestedFlavorStr.get(),
+                                          getter_AddRefs(item), &len);
+      if (NS_SUCCEEDED(rv)) {
+        aTransferable->SetTransferData(requestedFlavorStr.get(), item, len);
+        return NS_OK;
+      }
+
+      // Under some cases GetTransferData will return a failure, but the item
+      // will still have data that can be unwrapped and converted to an image.
+      if (NS_FAILED(rv) && item &&
+          TrySetTransferData(item, requestedFlavorStr.get(), aTransferable)) {
+        return NS_OK;
+      }
+    } else {
+      // Try to convert the stored transferable to the requested flavor.
+      mTransferable->GetTransferData(sourceFlavorStr.get(),
+                                     getter_AddRefs(item), &len);
+      if (TrySetTransferData(item, requestedFlavorStr.get(), aTransferable)) {
+        return NS_OK;
+      }
+    }
   }
-  nsCOMPtr<nsISupports> genericDataWrapper = do_QueryInterface(dataWrapper);
-  uint32_t len = mClipboard->GetText().Length() * sizeof(char16_t);
-  rv = aTransferable->SetTransferData(kUnicodeMime, genericDataWrapper, len);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
+
   return NS_OK;
 }
 
 NS_IMETHODIMP
 HeadlessClipboard::EmptyClipboard(int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
-  mClipboard->Clear();
+  mTransferable = nullptr;
   return NS_OK;
 }
 
 NS_IMETHODIMP
 HeadlessClipboard::HasDataMatchingFlavors(const char **aFlavorList,
-                                    uint32_t aLength, int32_t aWhichClipboard,
-                                    bool *aHasType)
+                                          uint32_t aLength,
+                                          int32_t aWhichClipboard,
+                                          bool *aHasType)
 {
   *aHasType = false;
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
+  if (!mTransferable) {
+    return NS_OK;
+  }
+
   // Retrieve the union of all aHasType in aFlavorList
   for (uint32_t i = 0; i < aLength; ++i) {
     const char *flavor = aFlavorList[i];
     if (!flavor) {
       continue;
     }
-    if (!strcmp(flavor, kUnicodeMime) && mClipboard->HasText()) {
+    nsAutoCString result;
+    if (GetSourceFlavor(flavor, result)) {
       *aHasType = true;
+      return NS_OK;
     }
   }
   return NS_OK;
 }
 
 NS_IMETHODIMP
 HeadlessClipboard::SupportsSelectionClipboard(bool *aIsSupported)
 {
--- a/widget/headless/HeadlessClipboard.h
+++ b/widget/headless/HeadlessClipboard.h
@@ -3,33 +3,37 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 
 #ifndef mozilla_widget_HeadlessClipboard_h
 #define mozilla_widget_HeadlessClipboard_h
 
 #include "nsIClipboard.h"
-#include "mozilla/UniquePtr.h"
-#include "HeadlessClipboardData.h"
+#include "nsCOMPtr.h"
 
 namespace mozilla {
 namespace widget {
 
 class HeadlessClipboard final : public nsIClipboard
 {
 public:
   NS_DECL_ISUPPORTS
   NS_DECL_NSICLIPBOARD
 
   HeadlessClipboard();
 
 protected:
   ~HeadlessClipboard() {}
 
 private:
-  UniquePtr<HeadlessClipboardData> mClipboard;
+
+  nsCOMPtr<nsITransferable> mTransferable;
+  // Checks if the stored transferable supports the requested flavor either
+  // by default or with conversion. On success the underlying flavor that can
+  // be used is stored in aSourceFlavor.
+  bool GetSourceFlavor(const char* aRequestedFlavor, nsACString &aSourceFlavor);
 };
 
 } // namespace widget
 } // namespace mozilla
 
 #endif
deleted file mode 100644
--- a/widget/headless/HeadlessClipboardData.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "HeadlessClipboardData.h"
-
-namespace mozilla {
-namespace widget {
-
-void
-HeadlessClipboardData::SetText(const nsAString &aText)
-{
-  mPlain = aText;
-}
-
-bool
-HeadlessClipboardData::HasText() const
-{
-  return !mPlain.IsEmpty();
-}
-
-const nsAString&
-HeadlessClipboardData::GetText() const
-{
-  return mPlain;
-}
-
-void
-HeadlessClipboardData::Clear()
-{
-  mPlain.Truncate(0);
-}
-
-} // namespace widget
-} // namespace mozilla
deleted file mode 100644
--- a/widget/headless/HeadlessClipboardData.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef mozilla_widget_HeadlessClipboardData_h
-#define mozilla_widget_HeadlessClipboardData_h
-
-#include "mozilla/RefPtr.h"
-#include "nsString.h"
-
-namespace mozilla {
-namespace widget {
-
-class HeadlessClipboardData final
-{
-public:
-  explicit HeadlessClipboardData() = default;
-  ~HeadlessClipboardData() = default;
-
-  // For text/plain
-  void SetText(const nsAString &aText);
-  bool HasText() const;
-  const nsAString& GetText() const;
-
-  // For other APIs
-  void Clear();
-
-private:
-  nsAutoString mPlain;
-};
-
-} // namespace widget
-} // namespace mozilla
-
-#endif // mozilla_widget_HeadlessClipboardData_h
--- a/widget/headless/moz.build
+++ b/widget/headless/moz.build
@@ -20,17 +20,16 @@ if widget_dir in ('gtk3', 'gtk2'):
     widget_dir = 'gtk'
 
 LOCAL_INCLUDES += [
     '/widget/%s' % widget_dir,
 ]
 
 UNIFIED_SOURCES += [
     'HeadlessClipboard.cpp',
-    'HeadlessClipboardData.cpp',
     'HeadlessCompositorWidget.cpp',
     'HeadlessScreenHelper.cpp',
     'HeadlessSound.cpp',
     'HeadlessWidget.cpp',
 ]
 
 if widget_dir == 'gtk':
     UNIFIED_SOURCES += [