Bug 1299068 - part 5: release/render buffers when VideoData sent to compositor.
MozReview-Commit-ID: JmEKLKlJnaL
--- a/dom/media/platforms/android/RemoteDataDecoder.cpp
+++ b/dom/media/platforms/android/RemoteDataDecoder.cpp
@@ -109,16 +109,50 @@ struct SampleTime final
int64_t mStart;
int64_t mDuration;
};
class RemoteVideoDecoder final : public RemoteDataDecoder
{
public:
+ // Hold an output buffer and render it to the surface when the frame is sent to compositor, or
+ // release it if not presented.
+ class RenderOrReleaseOutput : public VideoData::Listener
+ {
+ public:
+ RenderOrReleaseOutput(java::CodecProxy::Param aCodec, java::Sample::Param aSample)
+ : mCodec(aCodec),
+ mSample(aSample)
+ {}
+
+ ~RenderOrReleaseOutput()
+ {
+ ReleaseOutput(false);
+ }
+
+ void OnSentToCompositor() override
+ {
+ ReleaseOutput(true);
+ mCodec = nullptr;
+ mSample = nullptr;
+ }
+
+ private:
+ void ReleaseOutput(bool aToRender)
+ {
+ if (mCodec && mSample) {
+ mCodec->ReleaseOutput(mSample, aToRender);
+ }
+ }
+
+ java::CodecProxy::GlobalRef mCodec;
+ java::Sample::GlobalRef mSample;
+ };
+
class CallbacksSupport final : public JavaCallbacksSupport
{
public:
CallbacksSupport(RemoteVideoDecoder* aDecoder, MediaDataDecoderCallback* aCallback)
: JavaCallbacksSupport(aCallback)
, mDecoder(aDecoder)
{}
@@ -163,16 +197,19 @@ public:
durationUs.value(),
img,
!!(flags & MediaCodec::BUFFER_FLAG_SYNC_FRAME),
presentationTimeUs,
gfx::IntRect(0, 0,
mDecoder->mConfig.mDisplay.width,
mDecoder->mConfig.mDisplay.height));
+ UniquePtr<VideoData::Listener> listener(new RenderOrReleaseOutput(mDecoder->mJavaDecoder, aSample));
+ v->SetListener(Move(listener));
+
mDecoderCallback->Output(v);
}
if ((flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) != 0) {
mDecoderCallback->DrainComplete();
}
}
--- a/mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java
+++ b/mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java
@@ -13,27 +13,30 @@ import android.os.RemoteException;
import android.util.Log;
import android.view.Surface;
import org.mozilla.gecko.annotation.WrapForJNI;
import org.mozilla.gecko.mozglue.JNIObject;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
// Proxy class of ICodec binder.
public final class CodecProxy {
private static final String LOGTAG = "GeckoRemoteCodecProxy";
private static final boolean DEBUG = false;
private ICodec mRemote;
private FormatParam mFormat;
private Surface mOutputSurface;
private CallbacksForwarder mCallbacks;
private String mRemoteDrmStubId;
+ private Queue<Sample> mSurfaceOutputs = new ConcurrentLinkedQueue<>();
public interface Callbacks {
void onInputExhausted();
void onOutputFormatChanged(MediaFormat format);
void onOutput(Sample output);
void onError(boolean fatal);
}
@@ -62,19 +65,28 @@ public final class CodecProxy {
@Override
public void onOutputFormatChanged(FormatParam format) throws RemoteException {
mCallbacks.onOutputFormatChanged(format.asFormat());
}
@Override
public void onOutput(Sample sample) throws RemoteException {
- mCallbacks.onOutput(sample);
- mRemote.releaseOutput(sample, true);
- sample.dispose();
+ if (mOutputSurface != null) {
+ // Don't render to surface just yet. Callback will make that happen when it's time.
+ if (!sample.isEOS() || sample.info.size > 0) {
+ mSurfaceOutputs.offer(sample);
+ }
+ mCallbacks.onOutput(sample);
+ } else {
+ // Non-surface output needs no rendering.
+ mCallbacks.onOutput(sample);
+ mRemote.releaseOutput(sample, false);
+ sample.dispose();
+ }
}
@Override
public void onError(boolean fatal) throws RemoteException {
reportError(fatal);
}
private void reportError(boolean fatal) {
@@ -191,23 +203,63 @@ public final class CodecProxy {
@WrapForJNI
public synchronized boolean release() {
if (mRemote == null) {
Log.w(LOGTAG, "codec already ended");
return true;
}
if (DEBUG) Log.d(LOGTAG, "release " + this);
+
+ if (!mSurfaceOutputs.isEmpty()) {
+ // Flushing output buffers to surface may cause some frames to be skipped and
+ // should not happen unless caller release codec before processing all buffers.
+ Log.w(LOGTAG, "release codec when " + mSurfaceOutputs.size() + " output buffers unhandled");
+ try {
+ for (Sample s : mSurfaceOutputs) {
+ mRemote.releaseOutput(s, true);
+ }
+ } catch (RemoteException e) {
+ e.printStackTrace();
+ }
+ mSurfaceOutputs.clear();
+ }
+
try {
RemoteManager.getInstance().releaseCodec(this);
} catch (DeadObjectException e) {
return false;
} catch (RemoteException e) {
e.printStackTrace();
return false;
}
return true;
}
- public synchronized void reportError(boolean fatal) {
+ @WrapForJNI
+ public synchronized boolean releaseOutput(Sample sample, boolean render) {
+ if (!mSurfaceOutputs.remove(sample)) {
+ if (mRemote != null) Log.w(LOGTAG, "already released: " + sample);
+ return true;
+ }
+
+ if (mRemote == null) {
+ Log.w(LOGTAG, "codec already ended");
+ sample.dispose();
+ return true;
+ }
+
+ if (DEBUG && !render) Log.d(LOGTAG, "drop output:" + sample.info.presentationTimeUs);
+
+ try {
+ mRemote.releaseOutput(sample, render);
+ } catch (RemoteException e) {
+ e.printStackTrace();
+ }
+ sample.dispose();
+
+ return true;
+ }
+
+ /* package */ synchronized void reportError(boolean fatal) {
mCallbacks.reportError(fatal);
}
}
--- a/widget/android/fennec/FennecJNIWrappers.cpp
+++ b/widget/android/fennec/FennecJNIWrappers.cpp
@@ -216,16 +216,24 @@ auto CodecProxy::IsAdaptivePlaybackSuppo
constexpr char CodecProxy::Release_t::name[];
constexpr char CodecProxy::Release_t::signature[];
auto CodecProxy::Release() const -> bool
{
return mozilla::jni::Method<Release_t>::Call(CodecProxy::mCtx, nullptr);
}
+constexpr char CodecProxy::ReleaseOutput_t::name[];
+constexpr char CodecProxy::ReleaseOutput_t::signature[];
+
+auto CodecProxy::ReleaseOutput(mozilla::jni::Object::Param a0, bool a1) const -> bool
+{
+ return mozilla::jni::Method<ReleaseOutput_t>::Call(CodecProxy::mCtx, nullptr, a0, a1);
+}
+
const char CodecProxy::NativeCallbacks::name[] =
"org/mozilla/gecko/media/CodecProxy$NativeCallbacks";
constexpr char CodecProxy::NativeCallbacks::New_t::name[];
constexpr char CodecProxy::NativeCallbacks::New_t::signature[];
auto CodecProxy::NativeCallbacks::New() -> NativeCallbacks::LocalRef
{
--- a/widget/android/fennec/FennecJNIWrappers.h
+++ b/widget/android/fennec/FennecJNIWrappers.h
@@ -760,16 +760,37 @@ public:
static const mozilla::jni::CallingThread callingThread =
mozilla::jni::CallingThread::ANY;
static const mozilla::jni::DispatchTarget dispatchTarget =
mozilla::jni::DispatchTarget::CURRENT;
};
auto Release() const -> bool;
+ struct ReleaseOutput_t {
+ typedef CodecProxy Owner;
+ typedef bool ReturnType;
+ typedef bool SetterType;
+ typedef mozilla::jni::Args<
+ mozilla::jni::Object::Param,
+ bool> Args;
+ static constexpr char name[] = "releaseOutput";
+ static constexpr char signature[] =
+ "(Lorg/mozilla/gecko/media/Sample;Z)Z";
+ static const bool isStatic = false;
+ static const mozilla::jni::ExceptionMode exceptionMode =
+ mozilla::jni::ExceptionMode::ABORT;
+ static const mozilla::jni::CallingThread callingThread =
+ mozilla::jni::CallingThread::ANY;
+ static const mozilla::jni::DispatchTarget dispatchTarget =
+ mozilla::jni::DispatchTarget::CURRENT;
+ };
+
+ auto ReleaseOutput(mozilla::jni::Object::Param, bool) const -> bool;
+
static const mozilla::jni::CallingThread callingThread =
mozilla::jni::CallingThread::ANY;
};
class CodecProxy::NativeCallbacks : public mozilla::jni::ObjectBase<NativeCallbacks>
{
public: