Bug 1299068 - part 5: release/render buffers when VideoData sent to compositor. draft
authorJohn Lin <jolin@mozilla.com>
Wed, 30 Nov 2016 17:55:52 +0800
changeset 447862 f2cd1015801d57169e263497eba10096d3fd6e8c
parent 447861 17bb9dc3a203950d914d95068813ee35a90fa719
child 539156 d8e320c8520231fb60ca928e91579b15524a2789
push id38198
push userbmo:jolin@mozilla.com
push dateThu, 08 Dec 2016 04:52:32 +0000
bugs1299068
milestone53.0a1
Bug 1299068 - part 5: release/render buffers when VideoData sent to compositor. MozReview-Commit-ID: JmEKLKlJnaL
dom/media/platforms/android/RemoteDataDecoder.cpp
mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java
widget/android/fennec/FennecJNIWrappers.cpp
widget/android/fennec/FennecJNIWrappers.h
--- a/dom/media/platforms/android/RemoteDataDecoder.cpp
+++ b/dom/media/platforms/android/RemoteDataDecoder.cpp
@@ -109,16 +109,50 @@ struct SampleTime final
   int64_t mStart;
   int64_t mDuration;
 };
 
 
 class RemoteVideoDecoder final : public RemoteDataDecoder
 {
 public:
+  // Hold an output buffer and render it to the surface when the frame is sent to compositor, or
+  // release it if not presented.
+  class RenderOrReleaseOutput : public VideoData::Listener
+  {
+  public:
+    RenderOrReleaseOutput(java::CodecProxy::Param aCodec, java::Sample::Param aSample)
+      : mCodec(aCodec),
+        mSample(aSample)
+    {}
+
+    ~RenderOrReleaseOutput()
+    {
+      ReleaseOutput(false);
+    }
+
+    void OnSentToCompositor() override
+    {
+      ReleaseOutput(true);
+      mCodec = nullptr;
+      mSample = nullptr;
+    }
+
+  private:
+    void ReleaseOutput(bool aToRender)
+    {
+      if (mCodec && mSample) {
+        mCodec->ReleaseOutput(mSample, aToRender);
+      }
+    }
+
+    java::CodecProxy::GlobalRef mCodec;
+    java::Sample::GlobalRef mSample;
+  };
+
   class CallbacksSupport final : public JavaCallbacksSupport
   {
   public:
     CallbacksSupport(RemoteVideoDecoder* aDecoder, MediaDataDecoderCallback* aCallback)
       : JavaCallbacksSupport(aCallback)
       , mDecoder(aDecoder)
     {}
 
@@ -163,16 +197,19 @@ public:
                                     durationUs.value(),
                                     img,
                                     !!(flags & MediaCodec::BUFFER_FLAG_SYNC_FRAME),
                                     presentationTimeUs,
                                     gfx::IntRect(0, 0,
                                                   mDecoder->mConfig.mDisplay.width,
                                                   mDecoder->mConfig.mDisplay.height));
 
+        UniquePtr<VideoData::Listener> listener(new RenderOrReleaseOutput(mDecoder->mJavaDecoder, aSample));
+        v->SetListener(Move(listener));
+
         mDecoderCallback->Output(v);
       }
 
       if ((flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) != 0) {
         mDecoderCallback->DrainComplete();
       }
     }
 
--- a/mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java
+++ b/mobile/android/base/java/org/mozilla/gecko/media/CodecProxy.java
@@ -13,27 +13,30 @@ import android.os.RemoteException;
 import android.util.Log;
 import android.view.Surface;
 
 import org.mozilla.gecko.annotation.WrapForJNI;
 import org.mozilla.gecko.mozglue.JNIObject;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 // Proxy class of ICodec binder.
 public final class CodecProxy {
     private static final String LOGTAG = "GeckoRemoteCodecProxy";
     private static final boolean DEBUG = false;
 
     private ICodec mRemote;
     private FormatParam mFormat;
     private Surface mOutputSurface;
     private CallbacksForwarder mCallbacks;
     private String mRemoteDrmStubId;
+    private Queue<Sample> mSurfaceOutputs = new ConcurrentLinkedQueue<>();
 
     public interface Callbacks {
         void onInputExhausted();
         void onOutputFormatChanged(MediaFormat format);
         void onOutput(Sample output);
         void onError(boolean fatal);
     }
 
@@ -62,19 +65,28 @@ public final class CodecProxy {
 
         @Override
         public void onOutputFormatChanged(FormatParam format) throws RemoteException {
             mCallbacks.onOutputFormatChanged(format.asFormat());
         }
 
         @Override
         public void onOutput(Sample sample) throws RemoteException {
-            mCallbacks.onOutput(sample);
-            mRemote.releaseOutput(sample, true);
-            sample.dispose();
+            if (mOutputSurface != null) {
+                // Don't render to surface just yet. Callback will make that happen when it's time.
+                if (!sample.isEOS() || sample.info.size > 0) {
+                    mSurfaceOutputs.offer(sample);
+                }
+                mCallbacks.onOutput(sample);
+            } else {
+                // Non-surface output needs no rendering.
+                mCallbacks.onOutput(sample);
+                mRemote.releaseOutput(sample, false);
+                sample.dispose();
+            }
         }
 
         @Override
         public void onError(boolean fatal) throws RemoteException {
             reportError(fatal);
         }
 
         private void reportError(boolean fatal) {
@@ -191,23 +203,63 @@ public final class CodecProxy {
 
     @WrapForJNI
     public synchronized boolean release() {
         if (mRemote == null) {
             Log.w(LOGTAG, "codec already ended");
             return true;
         }
         if (DEBUG) Log.d(LOGTAG, "release " + this);
+
+        if (!mSurfaceOutputs.isEmpty()) {
+            // Flushing output buffers to surface may cause some frames to be skipped and
+            // should not happen unless caller release codec before processing all buffers.
+            Log.w(LOGTAG, "release codec when " + mSurfaceOutputs.size() + " output buffers unhandled");
+            try {
+                for (Sample s : mSurfaceOutputs) {
+                    mRemote.releaseOutput(s, true);
+                }
+            } catch (RemoteException e) {
+                e.printStackTrace();
+            }
+            mSurfaceOutputs.clear();
+        }
+
         try {
             RemoteManager.getInstance().releaseCodec(this);
         } catch (DeadObjectException e) {
             return false;
         } catch (RemoteException e) {
             e.printStackTrace();
             return false;
         }
         return true;
     }
 
-    public synchronized void reportError(boolean fatal) {
+    @WrapForJNI
+    public synchronized boolean releaseOutput(Sample sample, boolean render) {
+        if (!mSurfaceOutputs.remove(sample)) {
+            if (mRemote != null) Log.w(LOGTAG, "already released: " + sample);
+            return true;
+        }
+
+        if (mRemote == null) {
+            Log.w(LOGTAG, "codec already ended");
+            sample.dispose();
+            return true;
+        }
+
+        if (DEBUG && !render) Log.d(LOGTAG, "drop output:" + sample.info.presentationTimeUs);
+
+        try {
+            mRemote.releaseOutput(sample, render);
+        } catch (RemoteException e) {
+            e.printStackTrace();
+        }
+        sample.dispose();
+
+        return true;
+    }
+
+    /* package */ synchronized void reportError(boolean fatal) {
         mCallbacks.reportError(fatal);
     }
 }
--- a/widget/android/fennec/FennecJNIWrappers.cpp
+++ b/widget/android/fennec/FennecJNIWrappers.cpp
@@ -216,16 +216,24 @@ auto CodecProxy::IsAdaptivePlaybackSuppo
 constexpr char CodecProxy::Release_t::name[];
 constexpr char CodecProxy::Release_t::signature[];
 
 auto CodecProxy::Release() const -> bool
 {
     return mozilla::jni::Method<Release_t>::Call(CodecProxy::mCtx, nullptr);
 }
 
+constexpr char CodecProxy::ReleaseOutput_t::name[];
+constexpr char CodecProxy::ReleaseOutput_t::signature[];
+
+auto CodecProxy::ReleaseOutput(mozilla::jni::Object::Param a0, bool a1) const -> bool
+{
+    return mozilla::jni::Method<ReleaseOutput_t>::Call(CodecProxy::mCtx, nullptr, a0, a1);
+}
+
 const char CodecProxy::NativeCallbacks::name[] =
         "org/mozilla/gecko/media/CodecProxy$NativeCallbacks";
 
 constexpr char CodecProxy::NativeCallbacks::New_t::name[];
 constexpr char CodecProxy::NativeCallbacks::New_t::signature[];
 
 auto CodecProxy::NativeCallbacks::New() -> NativeCallbacks::LocalRef
 {
--- a/widget/android/fennec/FennecJNIWrappers.h
+++ b/widget/android/fennec/FennecJNIWrappers.h
@@ -760,16 +760,37 @@ public:
         static const mozilla::jni::CallingThread callingThread =
                 mozilla::jni::CallingThread::ANY;
         static const mozilla::jni::DispatchTarget dispatchTarget =
                 mozilla::jni::DispatchTarget::CURRENT;
     };
 
     auto Release() const -> bool;
 
+    struct ReleaseOutput_t {
+        typedef CodecProxy Owner;
+        typedef bool ReturnType;
+        typedef bool SetterType;
+        typedef mozilla::jni::Args<
+                mozilla::jni::Object::Param,
+                bool> Args;
+        static constexpr char name[] = "releaseOutput";
+        static constexpr char signature[] =
+                "(Lorg/mozilla/gecko/media/Sample;Z)Z";
+        static const bool isStatic = false;
+        static const mozilla::jni::ExceptionMode exceptionMode =
+                mozilla::jni::ExceptionMode::ABORT;
+        static const mozilla::jni::CallingThread callingThread =
+                mozilla::jni::CallingThread::ANY;
+        static const mozilla::jni::DispatchTarget dispatchTarget =
+                mozilla::jni::DispatchTarget::CURRENT;
+    };
+
+    auto ReleaseOutput(mozilla::jni::Object::Param, bool) const -> bool;
+
     static const mozilla::jni::CallingThread callingThread =
             mozilla::jni::CallingThread::ANY;
 
 };
 
 class CodecProxy::NativeCallbacks : public mozilla::jni::ObjectBase<NativeCallbacks>
 {
 public: