Bug 1265755 - Implement remote vp8 encoder and enable/disable them with pref; r?jolin
MozReview-Commit-ID: JZ1eUZNLAKq
--- a/dom/media/MediaPrefs.h
+++ b/dom/media/MediaPrefs.h
@@ -109,16 +109,17 @@ private:
DECL_MEDIA_PREF("media.use-blank-decoder", PDMUseBlankDecoder, bool, false);
DECL_MEDIA_PREF("media.gpu-process-decoder", PDMUseGPUDecoder, bool, false);
#ifdef MOZ_GONK_MEDIACODEC
DECL_MEDIA_PREF("media.gonk.enabled", PDMGonkDecoderEnabled, bool, true);
#endif
#ifdef MOZ_WIDGET_ANDROID
DECL_MEDIA_PREF("media.android-media-codec.enabled", PDMAndroidMediaCodecEnabled, bool, false);
DECL_MEDIA_PREF("media.android-media-codec.preferred", PDMAndroidMediaCodecPreferred, bool, false);
+ DECL_MEDIA_PREF("media.navigator.hardware.vp8_encode.acceleration_remote_enabled", RemoteMediaCodecVP8EncoderEnabled, bool, false);
#endif
#ifdef MOZ_FFMPEG
DECL_MEDIA_PREF("media.ffmpeg.enabled", PDMFFmpegEnabled, bool, true);
DECL_MEDIA_PREF("media.libavcodec.allow-obsolete", LibavcodecAllowObsolete, bool, false);
#endif
#if defined(MOZ_FFMPEG) || defined(MOZ_FFVPX)
DECL_MEDIA_PREF("media.ffmpeg.low-latency.enabled", PDMFFmpegLowLatencyEnabled, bool, false);
#endif
--- a/media/webrtc/signaling/src/media-conduit/MediaCodecVideoCodec.cpp
+++ b/media/webrtc/signaling/src/media-conduit/MediaCodecVideoCodec.cpp
@@ -2,25 +2,30 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "CSFLog.h"
#include "nspr.h"
#include "WebrtcMediaCodecVP8VideoCodec.h"
#include "MediaCodecVideoCodec.h"
+#include "MediaPrefs.h"
namespace mozilla {
static const char* logTag ="MediaCodecVideoCodec";
WebrtcVideoEncoder* MediaCodecVideoCodec::CreateEncoder(CodecType aCodecType) {
CSFLogDebug(logTag, "%s ", __FUNCTION__);
if (aCodecType == CODEC_VP8) {
- return new WebrtcMediaCodecVP8VideoEncoder();
+ if (MediaPrefs::RemoteMediaCodecVP8EncoderEnabled()) {
+ return new WebrtcMediaCodecVP8VideoRemoteEncoder();
+ } else {
+ return new WebrtcMediaCodecVP8VideoEncoder();
+ }
}
return nullptr;
}
WebrtcVideoDecoder* MediaCodecVideoCodec::CreateDecoder(CodecType aCodecType) {
CSFLogDebug(logTag, "%s ", __FUNCTION__);
if (aCodecType == CODEC_VP8) {
return new WebrtcMediaCodecVP8VideoDecoder();
--- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.cpp
+++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.cpp
@@ -4,44 +4,161 @@
#include <cstdio>
#include <iostream>
#include <queue>
#include "CSFLog.h"
#include "nspr.h"
+#include "JavaCallbacksSupport.h"
#include "MediaCodec.h"
#include "WebrtcMediaCodecVP8VideoCodec.h"
#include "AndroidJNIWrapper.h"
#include "mozilla/ArrayUtils.h"
#include "nsThreadUtils.h"
#include "mozilla/Monitor.h"
#include "runnable_utils.h"
+#include "MediaResult.h"
#include "AudioConduit.h"
#include "VideoConduit.h"
#include "libyuv/convert_from.h"
#include "libyuv/convert.h"
#include "libyuv/row.h"
#include "webrtc/modules/video_coding/include/video_error_codes.h"
+#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
#include <webrtc/common_video/libyuv/include/webrtc_libyuv.h>
using namespace mozilla;
+using namespace mozilla::java;
using namespace mozilla::java::sdk;
static const int32_t DECODER_TIMEOUT = 10 * PR_USEC_PER_MSEC; // 10ms
static const char MEDIACODEC_VIDEO_MIME_VP8[] = "video/x-vnd.on2.vp8";
namespace mozilla {
static const char* logTag ="WebrtcMediaCodecVP8VideoCodec";
+class CallbacksSupport final : public JavaCallbacksSupport
+{
+public:
+ CallbacksSupport(webrtc::EncodedImageCallback* aCallback) :
+ mCallback(aCallback),
+ mCritSect(webrtc::CriticalSectionWrapper::CreateCriticalSection()) {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ memset(&mEncodedImage, 0, sizeof(mEncodedImage));
+ }
+
+ ~CallbacksSupport() {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ if (mEncodedImage._size) {
+ delete [] mEncodedImage._buffer;
+ mEncodedImage._buffer = nullptr;
+ mEncodedImage._size = 0;
+ }
+ }
+
+ void VerifyAndAllocate(const uint32_t minimumSize)
+ {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ if(minimumSize > mEncodedImage._size)
+ {
+ uint8_t* newBuffer = new uint8_t[minimumSize];
+ MOZ_RELEASE_ASSERT(newBuffer);
+
+ if(mEncodedImage._buffer) {
+ delete [] mEncodedImage._buffer;
+ }
+ mEncodedImage._buffer = newBuffer;
+ mEncodedImage._size = minimumSize;
+ }
+ }
+
+ void HandleInputExhausted() override
+ {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ }
+
+ void HandleOutputFormatChanged(MediaFormat::Param aFormat) override
+ {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ }
+
+ void HandleOutput(Sample::Param aSample)
+ {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ BufferInfo::LocalRef info = aSample->Info();
+
+ int32_t size;
+ bool ok = NS_SUCCEEDED(info->Size(&size));
+ MOZ_RELEASE_ASSERT(ok);
+
+ if (size > 0) {
+ webrtc::CriticalSectionScoped lock(mCritSect.get());
+ VerifyAndAllocate(size);
+
+ int64_t presentationTimeUs;
+ ok = NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
+ MOZ_RELEASE_ASSERT(ok);
+
+ mEncodedImage._timeStamp = presentationTimeUs;
+ mEncodedImage.capture_time_ms_ = mEncodedImage._timeStamp;
+
+ int32_t flags;
+ ok = NS_SUCCEEDED(info->Flags(&flags));
+ MOZ_ASSERT(ok);
+
+ if (flags == MediaCodec::BUFFER_FLAG_SYNC_FRAME) {
+ mEncodedImage._frameType = webrtc::kVideoFrameKey;
+ } else {
+ mEncodedImage._frameType = webrtc::kVideoFrameDelta;
+ }
+ mEncodedImage._completeFrame = true;
+ mEncodedImage._length = size;
+
+ jni::ByteBuffer::LocalRef dest =
+ jni::ByteBuffer::New(mEncodedImage._buffer, size);
+ aSample->WriteToByteBuffer(dest);
+
+ webrtc::CodecSpecificInfo info;
+ info.codecType = webrtc::kVideoCodecVP8;
+ info.codecSpecific.VP8.pictureId = -1;
+ info.codecSpecific.VP8.tl0PicIdx = -1;
+ info.codecSpecific.VP8.keyIdx = -1;
+ info.codecSpecific.VP8.temporalIdx = 1;
+ info.codecSpecific.VP8.simulcastIdx = 0;
+
+ webrtc::RTPFragmentationHeader header;
+ memset(&header, 0, sizeof(header));
+ header.VerifyAndAllocateFragmentationHeader(1);
+ header.fragmentationLength[0] = mEncodedImage._length;
+
+ MOZ_RELEASE_ASSERT(mCallback);
+ mCallback->Encoded(mEncodedImage, &info, &header);
+ }
+ }
+
+ void HandleError(const MediaResult& aError) override
+ {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ }
+
+ friend class WebrtcMediaCodecVP8VideoRemoteEncoder;
+
+private:
+ webrtc::EncodedImageCallback* mCallback;
+ Atomic<bool> mCanceled;
+ webrtc::EncodedImage mEncodedImage;
+ rtc::scoped_ptr<webrtc::CriticalSectionWrapper> mCritSect;
+};
+
static MediaCodec::LocalRef CreateDecoder(const char* aMimeType)
{
if (!aMimeType) {
return nullptr;
}
MediaCodec::LocalRef codec;
MediaCodec::CreateDecoderByType(aMimeType, &codec);
@@ -203,21 +320,21 @@ public:
if (encoder) {
mCoder = CreateEncoder(mime);
if (NS_FAILED(res)) {
CSFLogDebug(logTag, "WebrtcAndroidMediaCodec::%s, CreateEncoderByType failed err = %d", __FUNCTION__, (int)res);
return NS_ERROR_FAILURE;
}
- res = format->SetInteger(nsCString("bitrate"), 1000*300);
- res = format->SetInteger(nsCString("bitrate-mode"), 2);
- res = format->SetInteger(nsCString("color-format"), 21);
- res = format->SetInteger(nsCString("frame-rate"), 30);
- res = format->SetInteger(nsCString("i-frame-interval"), 100);
+ res = format->SetInteger(MediaFormat::KEY_BIT_RATE, 1000*300);
+ res = format->SetInteger(MediaFormat::KEY_BITRATE_MODE, 2);
+ res = format->SetInteger(MediaFormat::KEY_COLOR_FORMAT, 21);
+ res = format->SetInteger(MediaFormat::KEY_FRAME_RATE, 30);
+ res = format->SetInteger(MediaFormat::KEY_I_FRAME_INTERVAL, 100);
} else {
mCoder = CreateDecoder(mime);
if (NS_FAILED(res)) {
CSFLogDebug(logTag, "WebrtcAndroidMediaCodec::%s, CreateDecoderByType failed err = %d", __FUNCTION__, (int)res);
return NS_ERROR_FAILURE;
}
}
@@ -854,16 +971,146 @@ int32_t WebrtcMediaCodecVP8VideoEncoder:
// XXX
// 1. implement MediaCodec's setParameters method
// 2.find a way to initiate a Java Bundle instance as parameter for MediaCodec setParameters method.
// mMediaCodecEncoder->setParameters
return WEBRTC_VIDEO_CODEC_OK;
}
+WebrtcMediaCodecVP8VideoRemoteEncoder::~WebrtcMediaCodecVP8VideoRemoteEncoder() {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+ Release();
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::InitEncode(
+ const webrtc::VideoCodec* codecSettings,
+ int32_t numberOfCores,
+ size_t maxPayloadSize) {
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::SetRates(uint32_t newBitRate, uint32_t frameRate) {
+ CSFLogDebug(logTag, "%s, newBitRate: %d, frameRate: %d", __FUNCTION__, newBitRate, frameRate);
+ if (!mJavaEncoder) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+ mJavaEncoder->SetRates(newBitRate);
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::Encode(
+ const webrtc::VideoFrame& inputImage,
+ const webrtc::CodecSpecificInfo* codecSpecificInfo,
+ const std::vector<webrtc::FrameType>* frame_types) {
+ CSFLogDebug(logTag, "%s, w = %d, h = %d", __FUNCTION__, inputImage.width(), inputImage.height());
+ if (inputImage.width() == 0 || inputImage.height() == 0) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ if (!mJavaEncoder) {
+ JavaCallbacksSupport::Init();
+ mJavaCallbacks = CodecProxy::NativeCallbacks::New();
+
+ JavaCallbacksSupport::AttachNative(
+ mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(mCallback));
+
+ MediaFormat::LocalRef format;
+
+ nsresult res = MediaFormat::CreateVideoFormat(nsCString(MEDIACODEC_VIDEO_MIME_VP8),
+ inputImage.width(),
+ inputImage.height(),
+ &format);
+
+ if (NS_FAILED(res)) {
+ CSFLogDebug(logTag, "%s, CreateVideoFormat failed err = %d", __FUNCTION__, (int)res);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ res = format->SetInteger(nsCString("bitrate"), 300 * 1000);
+ res = format->SetInteger(nsCString("bitrate-mode"), 2);
+ res = format->SetInteger(nsCString("color-format"), 21);
+ res = format->SetInteger(nsCString("frame-rate"), 30);
+ res = format->SetInteger(nsCString("i-frame-interval"), 100);
+
+ mJavaEncoder = CodecProxy::Create(true,
+ format,
+ nullptr,
+ mJavaCallbacks,
+ EmptyString());
+
+ if (mJavaEncoder == nullptr) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ }
+
+ size_t sizeY = inputImage.allocated_size(webrtc::kYPlane);
+ size_t sizeUV = inputImage.allocated_size(webrtc::kUPlane);
+ size_t size = sizeY + 2 * sizeUV;
+
+ if (mConvertBuf == nullptr) {
+ mConvertBuf = new uint8_t[size];
+ mConvertBufsize = size;
+ }
+
+ uint8_t* dstY = mConvertBuf;
+ uint16_t* dstUV = reinterpret_cast<uint16_t*>(dstY + sizeY);
+
+ bool converted = I420toNV12(dstY, dstUV, inputImage);
+ if (!converted) {
+ CSFLogError(logTag, "%s WebrtcMediaCodecVP8VideoEncoder::Encode() convert input buffer to NV12 error.", __FUNCTION__);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(mConvertBuf, size);
+
+ BufferInfo::LocalRef bufferInfo;
+ nsresult rv = BufferInfo::New(&bufferInfo);
+ if (NS_FAILED(rv)) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ if((*frame_types)[0] == webrtc::kVideoFrameKey) {
+ bufferInfo->Set(0, size, inputImage.timestamp(), MediaCodec::BUFFER_FLAG_SYNC_FRAME);
+ } else {
+ bufferInfo->Set(0, size, inputImage.timestamp(), 0);
+ }
+
+ mJavaEncoder->Input(bytes, bufferInfo, nullptr);
+
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) {
+ mCallback = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t WebrtcMediaCodecVP8VideoRemoteEncoder::Release() {
+ CSFLogDebug(logTag, "%s %p", __FUNCTION__, this);
+
+ if (mJavaEncoder) {
+ mJavaEncoder->Release();
+ mJavaEncoder = nullptr;
+ }
+
+ if (mJavaCallbacks) {
+ JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
+ JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
+ mJavaCallbacks = nullptr;
+ }
+
+ if (mConvertBuf) {
+ delete [] mConvertBuf;
+ mConvertBuf = nullptr;
+ }
+
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
// Decoder.
WebrtcMediaCodecVP8VideoDecoder::WebrtcMediaCodecVP8VideoDecoder()
: mCallback(nullptr)
, mFrameWidth(0)
, mFrameHeight(0)
, mMediaCodecDecoder(nullptr) {
CSFLogDebug(logTag, "%s ", __FUNCTION__);
}
--- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.h
+++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaCodecVP8VideoCodec.h
@@ -9,16 +9,17 @@
#include "mozilla/Mutex.h"
#include "nsThreadUtils.h"
#include "nsAutoPtr.h"
#include "MediaConduitInterface.h"
#include "AudioConduit.h"
#include "VideoConduit.h"
+#include "FennecJNIWrappers.h"
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
namespace mozilla {
struct EncodedFrame {
uint32_t width_;
uint32_t height_;
@@ -66,16 +67,49 @@ private:
uint32_t mFrameHeight;
WebrtcAndroidMediaCodec* mMediaCodecEncoder;
jobjectArray mInputBuffers;
jobjectArray mOutputBuffers;
};
+class WebrtcMediaCodecVP8VideoRemoteEncoder : public WebrtcVideoEncoder {
+public:
+ WebrtcMediaCodecVP8VideoRemoteEncoder() : mConvertBuf(nullptr), mConvertBufsize(0), mCallback(nullptr) {}
+
+ ~WebrtcMediaCodecVP8VideoRemoteEncoder() override;
+
+ // Implement VideoEncoder interface.
+ uint64_t PluginID() const override { return 0; }
+
+ int32_t InitEncode(const webrtc::VideoCodec* codecSettings,
+ int32_t numberOfCores,
+ size_t maxPayloadSize) override;
+
+ int32_t Encode(const webrtc::VideoFrame& inputImage,
+ const webrtc::CodecSpecificInfo* codecSpecificInfo,
+ const std::vector<webrtc::FrameType>* frame_types) override;
+
+ int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override;
+
+ int32_t Release() override;
+
+ int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) override { return 0; }
+
+ int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) override;
+
+private:
+ java::CodecProxy::GlobalRef mJavaEncoder;
+ java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks;
+ uint8_t* mConvertBuf;
+ uint8_t mConvertBufsize;
+ webrtc::EncodedImageCallback* mCallback;
+};
+
class WebrtcMediaCodecVP8VideoDecoder : public WebrtcVideoDecoder {
public:
WebrtcMediaCodecVP8VideoDecoder();
virtual ~WebrtcMediaCodecVP8VideoDecoder() override;
// Implement VideoDecoder interface.
virtual uint64_t PluginID() const override { return 0; }
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -509,16 +509,17 @@ pref("media.peerconnection.capture_delay
pref("media.getusermedia.playout_delay", 40);
pref("media.navigator.audio.full_duplex", true);
#elif defined(ANDROID)
pref("media.peerconnection.capture_delay", 100);
pref("media.getusermedia.playout_delay", 100);
pref("media.navigator.audio.full_duplex", true);
// Whether to enable Webrtc Hardware acceleration support
pref("media.navigator.hardware.vp8_encode.acceleration_enabled", false);
+pref("media.navigator.hardware.vp8_encode.acceleration_remote_enabled", false);
pref("media.navigator.hardware.vp8_decode.acceleration_enabled", false);
#elif defined(XP_LINUX)
pref("media.peerconnection.capture_delay", 70);
pref("media.getusermedia.playout_delay", 50);
pref("media.navigator.audio.full_duplex", true);
#else
// *BSD, others - merely a guess for now
pref("media.peerconnection.capture_delay", 50);