Bug 1470348 - Enable gfxVRExternal for Android; r=kip draft
authorKearwood Gilbert <kgilbert@mozilla.com>
Tue, 10 Jul 2018 17:41:58 -0700
changeset 816351 7103a4fdf3cb5934fb9e317c3028a3628cec71e7
parent 816350 b256d14602d36b9efe595472cc873e8170ed39f8
push id115807
push userbmo:kgilbert@mozilla.com
push dateWed, 11 Jul 2018 00:43:51 +0000
reviewerskip
bugs1470348
milestone63.0a1
Bug 1470348 - Enable gfxVRExternal for Android; r=kip MozReview-Commit-ID: 4XMdLLoedIh
dom/canvas/WebGLContext.cpp
gfx/layers/opengl/CompositorOGL.cpp
gfx/vr/external_api/moz_external_vr.h
gfx/vr/gfxVRExternal.cpp
gfx/vr/gfxVRExternal.h
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
--- a/dom/canvas/WebGLContext.cpp
+++ b/dom/canvas/WebGLContext.cpp
@@ -77,16 +77,20 @@
 #ifdef MOZ_WIDGET_COCOA
 #include "nsCocoaFeatures.h"
 #endif
 
 #ifdef XP_WIN
 #include "WGLLibrary.h"
 #endif
 
+#if defined(MOZ_WIDGET_ANDROID)
+    #include "../../gfx/vr/gfxVRExternal.h"
+#endif
+
 // Generated
 #include "mozilla/dom/WebGLRenderingContextBinding.h"
 
 
 namespace mozilla {
 
 using namespace mozilla::dom;
 using namespace mozilla::gfx;
@@ -2285,16 +2289,65 @@ WebGLContext::GetUnpackSize(bool isFunc3
 
     CheckedUint32 totalBytes = strideBytesPerImage * (usedImages - 1);
     totalBytes += strideBytesPerRow * (usedRowsPerImage - 1);
     totalBytes += usedBytesPerRow;
 
     return totalBytes;
 }
 
+
+#if defined(MOZ_WIDGET_ANDROID)
+already_AddRefed<layers::SharedSurfaceTextureClient>
+WebGLContext::GetVRFrame()
+{
+  if (IsContextLost()) {
+    ForceRestoreContext();
+  }
+
+  int frameId = gfx::impl::VRDisplayExternal::sPushIndex;
+  static int lastFrameId = -1;
+  /**
+   * Android doesn't like duplicated GetVRFrame within the same gfxVRExternal frame.
+   * Ballout forced composition calls if we are in the same VRExternal push frame index.
+   * Also discard frameId 0 because sometimes compositor is not paused yet due to channel communication delays.
+   */
+  const bool ignoreFrame = lastFrameId == frameId || frameId == 0;
+  lastFrameId = frameId;
+  if (!ignoreFrame) {
+      BeginComposition();
+      EndComposition();
+  }
+
+  if (IsContextLost()) {
+    return nullptr;
+  }
+
+  gl::GLScreenBuffer* screen = gl->Screen();
+  if (!screen) {
+    return nullptr;
+  }
+
+  RefPtr<SharedSurfaceTextureClient> sharedSurface = screen->Front();
+  if (!sharedSurface || !sharedSurface->Surf()) {
+    return nullptr;
+  }
+
+  /**
+   * Make sure that the WebGL buffer is committed to the attached SurfaceTexture on Android.
+   */
+  if (!ignoreFrame && !IsContextLost()) {
+    sharedSurface->Surf()->ProducerAcquire();
+    sharedSurface->Surf()->Commit();
+    sharedSurface->Surf()->ProducerRelease();
+  }
+
+  return sharedSurface.forget();
+}
+#else
 already_AddRefed<layers::SharedSurfaceTextureClient>
 WebGLContext::GetVRFrame()
 {
   /**
    * Swap buffers as though composition has occurred.
    * We will then share the resulting front buffer to be submitted to the VR
    * compositor.
    */
@@ -2310,16 +2363,18 @@ WebGLContext::GetVRFrame()
 
   RefPtr<SharedSurfaceTextureClient> sharedSurface = screen->Front();
   if (!sharedSurface)
       return nullptr;
 
   return sharedSurface.forget();
 }
 
+#endif  // ifdefined(MOZ_WIDGET_ANDROID)
+
 ////////////////////////////////////////////////////////////////////////////////
 
 static inline size_t
 SizeOfViewElem(const dom::ArrayBufferView& view)
 {
     const auto& elemType = view.Type();
     if (elemType == js::Scalar::MaxTypedArrayViewType) // DataViews.
         return 1;
--- a/gfx/layers/opengl/CompositorOGL.cpp
+++ b/gfx/layers/opengl/CompositorOGL.cpp
@@ -1841,18 +1841,20 @@ CompositorOGL::CopyToTarget(DrawTarget* 
 }
 
 void
 CompositorOGL::Pause()
 {
 #ifdef MOZ_WIDGET_ANDROID
   if (!gl() || gl()->IsDestroyed())
     return;
-
-  // ReleaseSurface internally calls MakeCurrent.
+  gl()->MakeCurrent();
+  java::GeckoSurfaceTexture::DestroyUnused((int64_t)mGLContext.get());
+  java::GeckoSurfaceTexture::DetachAllFromGLContext((int64_t)mGLContext.get());
+  // ReleaseSurface internally calls MakeCurrent
   gl()->ReleaseSurface();
 #endif
 }
 
 bool
 CompositorOGL::Resume()
 {
 #if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_UIKIT)
--- a/gfx/vr/external_api/moz_external_vr.h
+++ b/gfx/vr/external_api/moz_external_vr.h
@@ -43,16 +43,22 @@ static const uint32_t kVRGroupAll = 0xff
 
 static const int kVRDisplayNameMaxLen = 256;
 static const int kVRControllerNameMaxLen = 256;
 static const int kVRControllerMaxCount = 16;
 static const int kVRControllerMaxTriggers = 16;
 static const int kVRControllerMaxAxis = 16;
 static const int kVRLayerMaxCount = 8;
 
+#if defined(__ANDROID__)
+typedef uint64_t VRLayerTextureHandle;
+#else
+typedef void* VRLayerTextureHandle;
+#endif
+
 struct Point3D_POD
 {
   float x;
   float y;
   float z;
 };
 
 struct IntSize_POD
@@ -130,17 +136,17 @@ enum class VRDisplayCapabilityFlags : ui
   Cap_All = (1 << 9) - 1
 };
 
 #ifdef MOZILLA_INTERNAL_API
 MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(VRDisplayCapabilityFlags)
 #endif // MOZILLA_INTERNAL_API
 
 struct VRHMDSensorState {
-  int64_t inputFrameID;
+  uint64_t inputFrameID;
   double timestamp;
   VRDisplayCapabilityFlags flags;
 
   // These members will only change with inputFrameID:
   float orientation[4];
   float position[3];
   float leftViewMatrix[16];
   float rightViewMatrix[16];
@@ -272,29 +278,30 @@ enum class VRLayerType : uint16_t {
   LayerType_None = 0,
   LayerType_2D_Content = 1,
   LayerType_Stereo_Immersive = 2
 };
 
 enum class VRLayerTextureType : uint16_t {
   LayerTextureType_None = 0,
   LayerTextureType_D3D10SurfaceDescriptor = 1,
-  LayerTextureType_MacIOSurface = 2
+  LayerTextureType_MacIOSurface = 2,
+  LayerTextureType_GeckoSurfaceTexture = 3
 };
 
 struct VRLayer_2D_Content
 {
-  void* mTextureHandle;
+  VRLayerTextureHandle mTextureHandle;
   VRLayerTextureType mTextureType;
   uint64_t mFrameId;
 };
 
 struct VRLayer_Stereo_Immersive
 {
-  void* mTextureHandle;
+  VRLayerTextureHandle mTextureHandle;
   VRLayerTextureType mTextureType;
   uint64_t mFrameId;
   uint64_t mInputFrameId;
   VRLayerEyeRect mLeftEyeRect;
   VRLayerEyeRect mRightEyeRect;
 };
 
 struct VRLayerState
@@ -311,30 +318,31 @@ struct VRBrowserState
 #if defined(__ANDROID__)
   bool shutdown;
 #endif // defined(__ANDROID__)
   VRLayerState layerState[kVRLayerMaxCount];
 };
 
 struct VRSystemState
 {
-  uint32_t presentingGeneration;
   bool enumerationCompleted;
   VRDisplayState displayState;
   VRHMDSensorState sensorState;
   VRControllerState controllerState[kVRControllerMaxCount];
 };
 
 struct VRExternalShmem
 {
   int32_t version;
   int32_t size;
 #if defined(__ANDROID__)
   pthread_mutex_t systemMutex;
   pthread_mutex_t browserMutex;
+  pthread_cond_t systemCond;
+  pthread_cond_t browserCond;
 #else
   int64_t generationA;
 #endif // defined(__ANDROID__)
   VRSystemState state;
 #if !defined(__ANDROID__)
   int64_t generationB;
   int64_t browserGenerationA;
 #endif // !defined(__ANDROID__)
--- a/gfx/vr/gfxVRExternal.cpp
+++ b/gfx/vr/gfxVRExternal.cpp
@@ -49,16 +49,18 @@ static const char* kShmemName = "/moz.ge
 using namespace mozilla;
 using namespace mozilla::gfx;
 using namespace mozilla::gfx::impl;
 using namespace mozilla::layers;
 using namespace mozilla::dom;
 
 static const uint32_t kNumExternalHaptcs = 1;
 
+int VRDisplayExternal::sPushIndex = 0;
+
 VRDisplayExternal::VRDisplayExternal(const VRDisplayState& aDisplayState)
   : VRDisplayHost(VRDeviceType::External)
   , mIsPresenting(false)
   , mLastSensorState{}
 {
   MOZ_COUNT_CTOR_INHERITED(VRDisplayExternal, VRDisplayHost);
   mDisplayInfo.mDisplayState = aDisplayState;
 
@@ -101,16 +103,17 @@ VRDisplayExternal::GetSensorState()
 }
 
 void
 VRDisplayExternal::StartPresentation()
 {
   if (mIsPresenting) {
     return;
   }
+  sPushIndex = 0;
   mIsPresenting = true;
   mTelemetry.Clear();
   mTelemetry.mPresentationStart = TimeStamp::Now();
 
   // Indicate that we are ready to start immersive mode
   VRBrowserState state;
   memset(&state, 0, sizeof(VRBrowserState));
   state.layerState[0].type = VRLayerType::LayerType_Stereo_Immersive;
@@ -125,23 +128,24 @@ VRDisplayExternal::StartPresentation()
 
 void
 VRDisplayExternal::StopPresentation()
 {
   if (!mIsPresenting) {
     return;
   }
   mIsPresenting = false;
+  sPushIndex = 0;
 
   // Indicate that we have stopped immersive mode
   VRBrowserState state;
   memset(&state, 0, sizeof(VRBrowserState));
   VRManager *vm = VRManager::Get();
   VRSystemManagerExternal* manager = vm->GetExternalManager();
-  manager->PushState(&state);
+  manager->PushState(&state, true);
 
   // TODO - Implement telemetry:
 
 /*
   const TimeDuration duration = TimeStamp::Now() - mTelemetry.mPresentationStart;
   Telemetry::Accumulate(Telemetry::WEBVR_USERS_VIEW_IN, 2);
   Telemetry::Accumulate(Telemetry::WEBVR_TIME_SPENT_VIEWING_IN_OPENVR,
                         duration.ToMilliseconds());
@@ -152,17 +156,17 @@ VRDisplayExternal::StopPresentation()
                                         mTelemetry.mLastDroppedFrameCount) / duration.ToSeconds();
   Telemetry::Accumulate(Telemetry::WEBVR_DROPPED_FRAMES_IN_OPENVR, droppedFramesPerSec);
 */
 }
 
 bool
 VRDisplayExternal::PopulateLayerTexture(const layers::SurfaceDescriptor& aTexture,
                                         VRLayerTextureType* aTextureType,
-                                        void** aTextureHandle)
+                                        VRLayerTextureHandle* aTextureHandle)
 {
   switch (aTexture.type()) {
 #if defined(XP_WIN)
     case SurfaceDescriptor::TSurfaceDescriptorD3D10: {
       const SurfaceDescriptorD3D10& surf = aTexture.get_SurfaceDescriptorD3D10();
       *aTextureType = VRLayerTextureType::LayerTextureType_D3D10SurfaceDescriptor;
       *aTextureHandle = (void *)surf.handle();
       return true;
@@ -176,16 +180,28 @@ VRDisplayExternal::PopulateLayerTexture(
       if (!surf) {
         NS_WARNING("VRDisplayHost::SubmitFrame failed to get a MacIOSurface");
         return false;
       }
       *aTextureType = VRLayerTextureType::LayerTextureType_MacIOSurface;
       *aTextureHandle = (void *)surf->GetIOSurfacePtr();
       return true;
     }
+#elif defined(MOZ_WIDGET_ANDROID)
+    case SurfaceDescriptor::TSurfaceTextureDescriptor: {
+      const SurfaceTextureDescriptor& desc = aTexture.get_SurfaceTextureDescriptor();
+      java::GeckoSurfaceTexture::LocalRef surfaceTexture = java::GeckoSurfaceTexture::Lookup(desc.handle());
+      if (!surfaceTexture) {
+        NS_WARNING("VRDisplayHost::SubmitFrame failed to get a SurfaceTexture");
+        return false;
+      }
+      *aTextureType = VRLayerTextureType::LayerTextureType_GeckoSurfaceTexture;
+      *aTextureHandle = desc.handle();
+      return true;
+    }
 #endif
     default: {
       MOZ_ASSERT(false);
       return false;
     }
   }
 }
 
@@ -211,17 +227,18 @@ VRDisplayExternal::SubmitFrame(const lay
   layer.mLeftEyeRect.height = aLeftEyeRect.height;
   layer.mRightEyeRect.x = aRightEyeRect.x;
   layer.mRightEyeRect.y = aRightEyeRect.y;
   layer.mRightEyeRect.width = aRightEyeRect.width;
   layer.mRightEyeRect.height = aRightEyeRect.height;
 
   VRManager *vm = VRManager::Get();
   VRSystemManagerExternal* manager = vm->GetExternalManager();
-  manager->PushState(&state);
+  manager->PushState(&state, true);
+  sPushIndex++;
 
   VRDisplayState displayState;
   memset(&displayState, 0, sizeof(VRDisplayState));
   while (displayState.mLastSubmittedFrameId < aFrameId) {
     if (manager->PullState(&displayState, &mLastSensorState)) {
       if (!displayState.mIsConnected) {
         // Service has shut down or hardware has been disconnected
         return false;
@@ -335,22 +352,31 @@ VRSystemManagerExternal::OpenShmem()
     CloseShmem();
     return;
   }
 #elif defined(MOZ_WIDGET_ANDROID)
   mExternalShmem = (VRExternalShmem*)mozilla::GeckoVRManager::GetExternalContext();
   if (!mExternalShmem) {
     return;
   }
-  if (mExternalShmem->version != kVRExternalVersion) {
+  int32_t version = -1;
+  int32_t size = 0;
+  if (pthread_mutex_lock((pthread_mutex_t*)&(mExternalShmem->systemMutex)) == 0) {
+    version = mExternalShmem->version;
+    size = mExternalShmem->size;
+    pthread_mutex_unlock((pthread_mutex_t*)&(mExternalShmem->systemMutex));
+  } else {
+    return;
+  }
+  if (version != kVRExternalVersion) {
     mExternalShmem = nullptr;
     mExternalStructFailed = true;
     return;
   }
-  if (mExternalShmem->size != sizeof(VRExternalShmem)) {
+  if (size != sizeof(VRExternalShmem)) {
     mExternalShmem = nullptr;
     mExternalStructFailed = true;
     return;
   }
 #endif
   CheckForShutdown();
 }
 
@@ -368,19 +394,21 @@ VRSystemManagerExternal::CheckForShutdow
     }
   }
 #endif // defined(MOZ_WIDGET_ANDROID)
 }
 
 void
 VRSystemManagerExternal::CloseShmem()
 {
+#if !defined(MOZ_WIDGET_ANDROID)
   if (mSameProcess) {
     return;
   }
+#endif
 #if defined(XP_MACOSX)
   if (mExternalShmem) {
     munmap((void *)mExternalShmem, sizeof(VRExternalShmem));
     mExternalShmem = NULL;
   }
   if (mShmemFD) {
     close(mShmemFD);
   }
@@ -558,23 +586,23 @@ VRSystemManagerExternal::RemoveControlle
 bool
 VRSystemManagerExternal::PullState(VRDisplayState* aDisplayState, VRHMDSensorState* aSensorState /* = nullptr */)
 {
   bool success = false;
   MOZ_ASSERT(mExternalShmem);
   if (mExternalShmem) {
 #if defined(MOZ_WIDGET_ANDROID)
     if (pthread_mutex_lock((pthread_mutex_t*)&(mExternalShmem->systemMutex)) == 0) {
-        memcpy(aDisplayState, (void*)&(mExternalShmem->state.displayState), sizeof(VRDisplayState));
-        if (aSensorState) {
-          memcpy(aSensorState, (void*)&(mExternalShmem->state.sensorState), sizeof(VRHMDSensorState));
-        }
-        pthread_mutex_unlock((pthread_mutex_t*)&(mExternalShmem->systemMutex));
-        mDoShutdown = aDisplayState->shutdown;
-        success = mExternalShmem->state.enumerationCompleted;
+      memcpy(aDisplayState, (void*)&(mExternalShmem->state.displayState), sizeof(VRDisplayState));
+      if (aSensorState) {
+        memcpy(aSensorState, (void*)&(mExternalShmem->state.sensorState), sizeof(VRHMDSensorState));
+      }
+      success = mExternalShmem->state.enumerationCompleted;
+      pthread_mutex_unlock((pthread_mutex_t*)&(mExternalShmem->systemMutex));
+      mDoShutdown = aDisplayState->shutdown;
     }
 #else
     VRExternalShmem tmp;
     memcpy(&tmp, (void *)mExternalShmem, sizeof(VRExternalShmem));
     if (tmp.generationA == tmp.generationB && tmp.generationA != 0 && tmp.generationA != -1 && tmp.state.enumerationCompleted) {
       memcpy(aDisplayState, &tmp.state.displayState, sizeof(VRDisplayState));
       if (aSensorState) {
         memcpy(aSensorState, &tmp.state.sensorState, sizeof(VRHMDSensorState));
@@ -583,24 +611,27 @@ VRSystemManagerExternal::PullState(VRDis
     }
 #endif // defined(MOZ_WIDGET_ANDROID)
   }
 
   return success;
 }
 
 void
-VRSystemManagerExternal::PushState(VRBrowserState* aBrowserState)
+VRSystemManagerExternal::PushState(VRBrowserState* aBrowserState, bool aNotifyCond)
 {
   MOZ_ASSERT(aBrowserState);
   MOZ_ASSERT(mExternalShmem);
   if (mExternalShmem) {
 #if defined(MOZ_WIDGET_ANDROID)
     if (pthread_mutex_lock((pthread_mutex_t*)&(mExternalShmem->browserMutex)) == 0) {
       memcpy((void *)&(mExternalShmem->browserState), aBrowserState, sizeof(VRBrowserState));
+      if (aNotifyCond) {
+        pthread_cond_signal((pthread_cond_t*)&(mExternalShmem->browserCond));
+      }
       pthread_mutex_unlock((pthread_mutex_t*)&(mExternalShmem->browserMutex));
     }
 #else
     mExternalShmem->browserGenerationA++;
     memcpy((void *)&(mExternalShmem->browserState), (void *)aBrowserState, sizeof(VRBrowserState));
     mExternalShmem->browserGenerationB++;
 #endif // defined(MOZ_WIDGET_ANDROID)
   }
--- a/gfx/vr/gfxVRExternal.h
+++ b/gfx/vr/gfxVRExternal.h
@@ -26,16 +26,17 @@ namespace gfx {
 class VRThread;
 
 namespace impl {
 
 class VRDisplayExternal : public VRDisplayHost
 {
 public:
   void ZeroSensor() override;
+  static int sPushIndex;
 
 protected:
   VRHMDSensorState GetSensorState() override;
   void StartPresentation() override;
   void StopPresentation() override;
 
   bool SubmitFrame(const layers::SurfaceDescriptor& aTexture,
                    uint64_t aFrameId,
@@ -47,17 +48,17 @@ public:
   void Refresh();
 protected:
   virtual ~VRDisplayExternal();
   void Destroy();
 
 private:
   bool PopulateLayerTexture(const layers::SurfaceDescriptor& aTexture,
                             VRLayerTextureType* aTextureType,
-                            void** aTextureHandle);
+                            VRLayerTextureHandle* aTextureHandle);
 
   VRTelemetry mTelemetry;
   bool mIsPresenting;
   VRHMDSensorState mLastSensorState;
 };
 
 class VRControllerExternal : public VRControllerHost
 {
@@ -91,17 +92,17 @@ public:
   virtual void RemoveControllers() override;
   virtual void VibrateHaptic(uint32_t aControllerIdx,
                              uint32_t aHapticIndex,
                              double aIntensity,
                              double aDuration,
                              const VRManagerPromise& aPromise) override;
   virtual void StopVibrateHaptic(uint32_t aControllerIdx) override;
   bool PullState(VRDisplayState* aDisplayState, VRHMDSensorState* aSensorState = nullptr);
-  void PushState(VRBrowserState* aBrowserState);
+  void PushState(VRBrowserState* aBrowserState, const bool aNotifyCond = false);
 
 protected:
   explicit VRSystemManagerExternal(VRExternalShmem* aAPIShmem = nullptr);
   virtual ~VRSystemManagerExternal();
 
 private:
   // there can only be one
   RefPtr<impl::VRDisplayExternal> mDisplay;
--- a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
@@ -204,16 +204,31 @@ import org.mozilla.gecko.annotation.Wrap
                     Log.e(LOGTAG, "Failed to finalize SurfaceTexture", t);
                 }
             } catch (Exception e) {
                 Log.e(LOGTAG, "Failed to destroy SurfaceTexture", e);
             }
         }
     }
 
+    @WrapForJNI
+    public static void detachAllFromGLContext(long context) {
+        synchronized (sSurfaceTextures) {
+            for (GeckoSurfaceTexture tex : sSurfaceTextures.values()) {
+                try {
+                    if (tex.isAttachedToGLContext(context)) {
+                        tex.detachFromGLContext();
+                    }
+                } catch (Exception e) {
+                    Log.e(LOGTAG, "Failed to detach SurfaceTexture with handle: " + tex.mHandle, e);
+                }
+            }
+        }
+    }
+
     public static GeckoSurfaceTexture acquire(boolean singleBufferMode) {
         if (singleBufferMode && !isSingleBufferSupported()) {
             throw new IllegalArgumentException("single buffer mode not supported on API version < 19");
         }
 
         synchronized (sSurfaceTextures) {
             // We want to limit the maximum number of SurfaceTextures at any one time.
             // This is because they use a large number of fds, and once the process' limit