Bug 1265755 - Implement remote vp8 encoder and enable/disable them with pref; r?jolin draft
authorMunro Mengjue Chiang <mchiang@mozilla.com>
Thu, 30 Mar 2017 09:00:13 +0800
changeset 554379 1428c9291ee885a9baa612989dd0c58df0a13c00
parent 554275 25e80ccf3d9a39d2d761ed2db5c5ff927ca6bfd6
child 622307 521ab09dd6bc7d5b61128745dfa5ca98802d43c2
push id51901
push userbmo:mchiang@mozilla.com
push dateFri, 31 Mar 2017 10:17:48 +0000
reviewersjolin
bugs1265755
milestone55.0a1
Bug 1265755 - Implement remote vp8 encoder and enable/disable them with pref; r?jolin MozReview-Commit-ID: JZ1eUZNLAKq
dom/media/MediaPrefs.h
media/webrtc/signaling/src/media-conduit/MediaCodecVideoCodec.cpp
media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.cpp
media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.h
modules/libpref/init/all.js
--- a/dom/media/MediaPrefs.h
+++ b/dom/media/MediaPrefs.h
@@ -109,16 +109,17 @@ private:
   DECL_MEDIA_PREF("media.use-blank-decoder",                  PDMUseBlankDecoder, bool, false);
   DECL_MEDIA_PREF("media.gpu-process-decoder",                PDMUseGPUDecoder, bool, false);
 #ifdef MOZ_GONK_MEDIACODEC
   DECL_MEDIA_PREF("media.gonk.enabled",                       PDMGonkDecoderEnabled, bool, true);
 #endif
 #ifdef MOZ_WIDGET_ANDROID
   DECL_MEDIA_PREF("media.android-media-codec.enabled",        PDMAndroidMediaCodecEnabled, bool, false);
   DECL_MEDIA_PREF("media.android-media-codec.preferred",      PDMAndroidMediaCodecPreferred, bool, false);
+  DECL_MEDIA_PREF("media.navigator.hardware.vp8_encode.acceleration_remote_enabled", RemoteMediaCodecVP8EncoderEnabled, bool, false);
 #endif
 #ifdef MOZ_FFMPEG
   DECL_MEDIA_PREF("media.ffmpeg.enabled",                     PDMFFmpegEnabled, bool, true);
   DECL_MEDIA_PREF("media.libavcodec.allow-obsolete",          LibavcodecAllowObsolete, bool, false);
 #endif
 #if defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)
   DECL_MEDIA_PREF("media.ffmpeg.low-latency.enabled",         PDMFFmpegLowLatencyEnabled, bool, false);
 #endif
--- a/media/webrtc/signaling/src/media-conduit/MediaCodecVideoCodec.cpp
+++ b/media/webrtc/signaling/src/media-conduit/MediaCodecVideoCodec.cpp
@@ -2,25 +2,30 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "CSFLog.h"
 #include "nspr.h"
 
 #include "WebrtcMediaCodecVP8VideoCodec.h"
 #include "MediaCodecVideoCodec.h"
+#include "MediaPrefs.h"
 
 namespace mozilla {
 
 static const char* logTag ="MediaCodecVideoCodec";
 
 WebrtcVideoEncoder* MediaCodecVideoCodec::CreateEncoder(CodecType aCodecType) {
   CSFLogDebug(logTag,  "%s ", __FUNCTION__);
   if (aCodecType == CODEC_VP8) {
-     return new WebrtcMediaCodecVP8VideoEncoder();
+    if (MediaPrefs::RemoteMediaCodecVP8EncoderEnabled()) {
+      return new WebrtcMediaCodecVP8VideoRemoteEncoder();
+    } else {
+      return new WebrtcMediaCodecVP8VideoEncoder();
+    }
   }
   return nullptr;
 }
 
 WebrtcVideoDecoder* MediaCodecVideoCodec::CreateDecoder(CodecType aCodecType) {
   CSFLogDebug(logTag,  "%s ", __FUNCTION__);
   if (aCodecType == CODEC_VP8) {
     return new WebrtcMediaCodecVP8VideoDecoder();
--- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.cpp
+++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.cpp
@@ -4,44 +4,161 @@
 
 #include <cstdio>
 #include <iostream>
 #include <queue>
 
 #include "CSFLog.h"
 #include "nspr.h"
 
+#include "JavaCallbacksSupport.h"
 #include "MediaCodec.h"
 #include "WebrtcMediaCodecVP8VideoCodec.h"
 #include "AndroidJNIWrapper.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Monitor.h"
 #include "runnable_utils.h"
+#include "MediaResult.h"
 
 #include "AudioConduit.h"
 #include "VideoConduit.h"
 #include "libyuv/convert_from.h"
 #include "libyuv/convert.h"
 #include "libyuv/row.h"
 
 #include "webrtc/modules/video_coding/include/video_error_codes.h"
+#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
 
 #include <webrtc/common_video/libyuv/include/webrtc_libyuv.h>
 
 using namespace mozilla;
+using namespace mozilla::java;
 using namespace mozilla::java::sdk;
 
 static const int32_t DECODER_TIMEOUT = 10 * PR_USEC_PER_MSEC; // 10ms
 static const char MEDIACODEC_VIDEO_MIME_VP8[] = "video/x-vnd.on2.vp8";
 
 namespace mozilla {
 
 static const char* logTag ="WebrtcMediaCodecVP8VideoCodec";
 
+class CallbacksSupport final : public JavaCallbacksSupport
+{
+public:
+  CallbacksSupport(webrtc::EncodedImageCallback* aCallback) :
+    mCallback(aCallback),
+    mCritSect(webrtc::CriticalSectionWrapper::CreateCriticalSection()) {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+    memset(&mEncodedImage, 0, sizeof(mEncodedImage));
+  }
+
+  ~CallbacksSupport() {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+    if (mEncodedImage._size) {
+      delete [] mEncodedImage._buffer;
+      mEncodedImage._buffer = nullptr;
+      mEncodedImage._size = 0;
+    }
+  }
+
+  void VerifyAndAllocate(const uint32_t minimumSize)
+  {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+    if(minimumSize > mEncodedImage._size)
+    {
+        uint8_t* newBuffer = new uint8_t[minimumSize];
+        MOZ_RELEASE_ASSERT(newBuffer);
+
+        if(mEncodedImage._buffer) {
+            delete [] mEncodedImage._buffer;
+        }
+        mEncodedImage._buffer = newBuffer;
+        mEncodedImage._size = minimumSize;
+    }
+  }
+
+  void HandleInputExhausted() override
+  {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+  }
+
+  void HandleOutputFormatChanged(MediaFormat::Param aFormat) override
+  {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+  }
+
+  void HandleOutput(Sample::Param aSample)
+  {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+    BufferInfo::LocalRef info = aSample->Info();
+
+    int32_t size;
+    bool ok = NS_SUCCEEDED(info->Size(&size));
+    MOZ_RELEASE_ASSERT(ok);
+
+    if (size > 0) {
+      webrtc::CriticalSectionScoped lock(mCritSect.get());
+      VerifyAndAllocate(size);
+
+      int64_t presentationTimeUs;
+      ok = NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
+      MOZ_RELEASE_ASSERT(ok);
+
+      mEncodedImage._timeStamp = presentationTimeUs;
+      mEncodedImage.capture_time_ms_ = mEncodedImage._timeStamp;
+
+      int32_t flags;
+      ok = NS_SUCCEEDED(info->Flags(&flags));
+      MOZ_ASSERT(ok);
+
+      if (flags == MediaCodec::BUFFER_FLAG_SYNC_FRAME) {
+        mEncodedImage._frameType = webrtc::kVideoFrameKey;
+      } else {
+        mEncodedImage._frameType = webrtc::kVideoFrameDelta;
+      }
+      mEncodedImage._completeFrame = true;
+      mEncodedImage._length = size;
+
+      jni::ByteBuffer::LocalRef dest =
+        jni::ByteBuffer::New(mEncodedImage._buffer, size);
+      aSample->WriteToByteBuffer(dest);
+
+      webrtc::CodecSpecificInfo info;
+      info.codecType = webrtc::kVideoCodecVP8;
+      info.codecSpecific.VP8.pictureId = -1;
+      info.codecSpecific.VP8.tl0PicIdx = -1;
+      info.codecSpecific.VP8.keyIdx = -1;
+      info.codecSpecific.VP8.temporalIdx = 1;
+      info.codecSpecific.VP8.simulcastIdx = 0;
+
+      webrtc::RTPFragmentationHeader header;
+      memset(&header, 0, sizeof(header));
+      header.VerifyAndAllocateFragmentationHeader(1);
+      header.fragmentationLength[0] = mEncodedImage._length;
+
+      MOZ_RELEASE_ASSERT(mCallback);
+      mCallback->Encoded(mEncodedImage, &info, &header);
+    }
+  }
+
+  void HandleError(const MediaResult& aError) override
+  {
+    CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+  }
+
+  friend class WebrtcMediaCodecVP8VideoRemoteEncoder;
+
+private:
+  webrtc::EncodedImageCallback* mCallback;
+  Atomic<bool> mCanceled;
+  webrtc::EncodedImage mEncodedImage;
+  rtc::scoped_ptr<webrtc::CriticalSectionWrapper> mCritSect;
+};
+
 static MediaCodec::LocalRef CreateDecoder(const char* aMimeType)
 {
   if (!aMimeType) {
     return nullptr;
   }
 
   MediaCodec::LocalRef codec;
   MediaCodec::CreateDecoderByType(aMimeType, &codec);
@@ -203,21 +320,21 @@ public:
       if (encoder) {
         mCoder = CreateEncoder(mime);
 
         if (NS_FAILED(res)) {
           CSFLogDebug(logTag, "WebrtcAndroidMediaCodec::%s, CreateEncoderByType failed err = %d", __FUNCTION__, (int)res);
           return NS_ERROR_FAILURE;
         }
 
-        res = format->SetInteger(nsCString("bitrate"), 1000*300);
-        res = format->SetInteger(nsCString("bitrate-mode"), 2);
-        res = format->SetInteger(nsCString("color-format"), 21);
-        res = format->SetInteger(nsCString("frame-rate"), 30);
-        res = format->SetInteger(nsCString("i-frame-interval"), 100);
+        res = format->SetInteger(MediaFormat::KEY_BIT_RATE, 1000*300);
+        res = format->SetInteger(MediaFormat::KEY_BITRATE_MODE, 2);
+        res = format->SetInteger(MediaFormat::KEY_COLOR_FORMAT, 21);
+        res = format->SetInteger(MediaFormat::KEY_FRAME_RATE, 30);
+        res = format->SetInteger(MediaFormat::KEY_I_FRAME_INTERVAL, 100);
 
       } else {
         mCoder = CreateDecoder(mime);
         if (NS_FAILED(res)) {
           CSFLogDebug(logTag, "WebrtcAndroidMediaCodec::%s, CreateDecoderByType failed err = %d", __FUNCTION__, (int)res);
           return NS_ERROR_FAILURE;
         }
       }
@@ -854,16 +971,146 @@ int32_t WebrtcMediaCodecVP8VideoEncoder:
   // XXX
   // 1. implement MediaCodec's setParameters method
   // 2.find a way to initiate a Java Bundle instance as parameter for MediaCodec setParameters method.
   // mMediaCodecEncoder->setParameters
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+WebrtcMediaCodecVP8VideoRemoteEncoder::~WebrtcMediaCodecVP8VideoRemoteEncoder() {
+  CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+  Release();
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::InitEncode(
+    const webrtc::VideoCodec* codecSettings,
+    int32_t numberOfCores,
+    size_t maxPayloadSize) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::SetRates(uint32_t newBitRate, uint32_t frameRate) {
+  CSFLogDebug(logTag,  "%s, newBitRate: %d, frameRate: %d", __FUNCTION__, newBitRate, frameRate);
+  if (!mJavaEncoder) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  mJavaEncoder->SetRates(newBitRate);
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::Encode(
+    const webrtc::VideoFrame& inputImage,
+    const webrtc::CodecSpecificInfo* codecSpecificInfo,
+    const std::vector<webrtc::FrameType>* frame_types) {
+  CSFLogDebug(logTag,  "%s, w = %d, h = %d", __FUNCTION__, inputImage.width(), inputImage.height());
+  if (inputImage.width() == 0 || inputImage.height() == 0) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  if (!mJavaEncoder) {
+    JavaCallbacksSupport::Init();
+    mJavaCallbacks = CodecProxy::NativeCallbacks::New();
+
+    JavaCallbacksSupport::AttachNative(
+      mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(mCallback));
+
+    MediaFormat::LocalRef format;
+
+    nsresult res = MediaFormat::CreateVideoFormat(nsCString(MEDIACODEC_VIDEO_MIME_VP8),
+                                                  inputImage.width(),
+                                                  inputImage.height(),
+                                                  &format);
+
+    if (NS_FAILED(res)) {
+      CSFLogDebug(logTag, "%s, CreateVideoFormat failed err = %d", __FUNCTION__, (int)res);
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
+    res = format->SetInteger(nsCString("bitrate"), 300 * 1000);
+    res = format->SetInteger(nsCString("bitrate-mode"), 2);
+    res = format->SetInteger(nsCString("color-format"), 21);
+    res = format->SetInteger(nsCString("frame-rate"), 30);
+    res = format->SetInteger(nsCString("i-frame-interval"), 100);
+
+    mJavaEncoder = CodecProxy::Create(true,
+                                      format,
+                                      nullptr,
+                                      mJavaCallbacks,
+                                      EmptyString());
+
+    if (mJavaEncoder == nullptr) {
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+  }
+
+  size_t sizeY = inputImage.allocated_size(webrtc::kYPlane);
+  size_t sizeUV = inputImage.allocated_size(webrtc::kUPlane);
+  size_t size = sizeY + 2 * sizeUV;
+
+  if (mConvertBuf == nullptr) {
+    mConvertBuf = new uint8_t[size];
+    mConvertBufsize = size;
+  }
+
+  uint8_t* dstY = mConvertBuf;
+  uint16_t* dstUV = reinterpret_cast<uint16_t*>(dstY + sizeY);
+
+  bool converted = I420toNV12(dstY, dstUV, inputImage);
+  if (!converted) {
+    CSFLogError(logTag,  "%s WebrtcMediaCodecVP8VideoEncoder::Encode() convert input buffer to NV12 error.", __FUNCTION__);
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(mConvertBuf, size);
+
+  BufferInfo::LocalRef bufferInfo;
+  nsresult rv = BufferInfo::New(&bufferInfo);
+  if (NS_FAILED(rv)) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  if((*frame_types)[0] == webrtc::kVideoFrameKey) {
+    bufferInfo->Set(0, size, inputImage.timestamp(), MediaCodec::BUFFER_FLAG_SYNC_FRAME);
+  } else {
+    bufferInfo->Set(0, size, inputImage.timestamp(), 0);
+  }
+
+  mJavaEncoder->Input(bytes, bufferInfo, nullptr);
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) {
+  mCallback = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::Release() {
+  CSFLogDebug(logTag,  "%s %p", __FUNCTION__, this);
+
+  if (mJavaEncoder) {
+    mJavaEncoder->Release();
+    mJavaEncoder = nullptr;
+  }
+
+  if (mJavaCallbacks) {
+    JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
+    JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
+    mJavaCallbacks = nullptr;
+  }
+
+  if (mConvertBuf) {
+    delete [] mConvertBuf;
+    mConvertBuf = nullptr;
+  }
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
 // Decoder.
 WebrtcMediaCodecVP8VideoDecoder::WebrtcMediaCodecVP8VideoDecoder()
   : mCallback(nullptr)
   , mFrameWidth(0)
   , mFrameHeight(0)
   , mMediaCodecDecoder(nullptr) {
   CSFLogDebug(logTag,  "%s ", __FUNCTION__);
 }
--- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.h
+++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.h
@@ -9,16 +9,17 @@
 
 #include "mozilla/Mutex.h"
 #include "nsThreadUtils.h"
 #include "nsAutoPtr.h"
 
 #include "MediaConduitInterface.h"
 #include "AudioConduit.h"
 #include "VideoConduit.h"
+#include "FennecJNIWrappers.h"
 
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 
 namespace mozilla {
 
 struct EncodedFrame {
   uint32_t width_;
   uint32_t height_;
@@ -66,16 +67,49 @@ private:
   uint32_t mFrameHeight;
 
   WebrtcAndroidMediaCodec* mMediaCodecEncoder;
 
   jobjectArray mInputBuffers;
   jobjectArray mOutputBuffers;
 };
 
+class WebrtcMediaCodecVP8VideoRemoteEncoder : public WebrtcVideoEncoder {
+public:
+  WebrtcMediaCodecVP8VideoRemoteEncoder() : mConvertBuf(nullptr), mConvertBufsize(0), mCallback(nullptr) {}
+
+  ~WebrtcMediaCodecVP8VideoRemoteEncoder() override;
+
+  // Implement VideoEncoder interface.
+  uint64_t PluginID() const override { return 0; }
+
+  int32_t InitEncode(const webrtc::VideoCodec* codecSettings,
+                     int32_t numberOfCores,
+                     size_t maxPayloadSize) override;
+
+  int32_t Encode(const webrtc::VideoFrame& inputImage,
+                 const webrtc::CodecSpecificInfo* codecSpecificInfo,
+                 const std::vector<webrtc::FrameType>* frame_types) override;
+
+  int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override;
+
+  int32_t Release() override;
+
+  int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) override { return 0; }
+
+  int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) override;
+
+private:
+  java::CodecProxy::GlobalRef mJavaEncoder;
+  java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks;
+  uint8_t* mConvertBuf;
+  uint8_t mConvertBufsize;
+  webrtc::EncodedImageCallback* mCallback;
+};
+
 class WebrtcMediaCodecVP8VideoDecoder : public WebrtcVideoDecoder {
 public:
   WebrtcMediaCodecVP8VideoDecoder();
 
   virtual ~WebrtcMediaCodecVP8VideoDecoder() override;
 
   // Implement VideoDecoder interface.
   virtual uint64_t PluginID() const override { return 0; }
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -509,16 +509,17 @@ pref("media.peerconnection.capture_delay
 pref("media.getusermedia.playout_delay", 40);
 pref("media.navigator.audio.full_duplex", true);
 #elif defined(ANDROID)
 pref("media.peerconnection.capture_delay", 100);
 pref("media.getusermedia.playout_delay", 100);
 pref("media.navigator.audio.full_duplex", true);
 // Whether to enable Webrtc Hardware acceleration support
 pref("media.navigator.hardware.vp8_encode.acceleration_enabled", false);
+pref("media.navigator.hardware.vp8_encode.acceleration_remote_enabled", false);
 pref("media.navigator.hardware.vp8_decode.acceleration_enabled", false);
 #elif defined(XP_LINUX)
 pref("media.peerconnection.capture_delay", 70);
 pref("media.getusermedia.playout_delay", 50);
 pref("media.navigator.audio.full_duplex", true);
 #else
 // *BSD, others - merely a guess for now
 pref("media.peerconnection.capture_delay", 50);