Bug 1232043 - Reconfigure VP8 encoder when resolution change. r?pehrsons draft
authorAlex Chronopoulos <achronop@gmail.com>
Fri, 24 Mar 2017 13:58:32 +0200
changeset 504691 f58ebed961878eb41243ef32eb3164c78ced6fd6
parent 504562 72bc265f157f3937f25d18be68b60fb982df695a
child 504692 6727ddff36d99d93fd028fe3b99c9fff86387a9c
push id50852
push userachronop@gmail.com
push dateFri, 24 Mar 2017 15:56:03 +0000
reviewerspehrsons
bugs1232043
milestone55.0a1
Bug 1232043 - Reconfigure VP8 encoder when resolution change. r?pehrsons MozReview-Commit-ID: AtjRgWlJhtI
dom/media/encoder/VP8TrackEncoder.cpp
dom/media/encoder/VP8TrackEncoder.h
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -59,96 +59,61 @@ VP8TrackEncoder::VP8TrackEncoder(TrackRa
   , mVPXContext(new vpx_codec_ctx_t())
   , mVPXImageWrapper(new vpx_image_t())
 {
   MOZ_COUNT_CTOR(VP8TrackEncoder);
 }
 
 VP8TrackEncoder::~VP8TrackEncoder()
 {
+  Destroy();
+  MOZ_COUNT_DTOR(VP8TrackEncoder);
+}
+
+void
+VP8TrackEncoder::Destroy()
+{
+  ReentrantMonitorAutoEnter mon(mReentrantMonitor);
   if (mInitialized) {
     vpx_codec_destroy(mVPXContext);
   }
 
   if (mVPXImageWrapper) {
     vpx_img_free(mVPXImageWrapper);
   }
-  MOZ_COUNT_DTOR(VP8TrackEncoder);
+  mInitialized = false;
 }
 
 nsresult
 VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
                       int32_t aDisplayHeight)
 {
   if (aWidth < 1 || aHeight < 1 || aDisplayWidth < 1 || aDisplayHeight < 1) {
     return NS_ERROR_FAILURE;
   }
 
   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+  if (mInitialized) {
+    MOZ_ASSERT(false);
+    return NS_ERROR_FAILURE;
+  }
 
-  mFrameWidth = aWidth;
-  mFrameHeight = aHeight;
-  mDisplayWidth = aDisplayWidth;
-  mDisplayHeight = aDisplayHeight;
+
 
   // Encoder configuration structure.
   vpx_codec_enc_cfg_t config;
-  memset(&config, 0, sizeof(vpx_codec_enc_cfg_t));
-  if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config, 0)) {
-    return NS_ERROR_FAILURE;
-  }
+  nsresult rv = SetConfigurationValues(aWidth, aHeight, aDisplayWidth, aDisplayHeight, config);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
 
   // Creating a wrapper to the image - setting image data to NULL. Actual
   // pointer will be set in encode. Setting align to 1, as it is meaningless
   // (actual memory is not allocated).
   vpx_img_wrap(mVPXImageWrapper, VPX_IMG_FMT_I420,
                mFrameWidth, mFrameHeight, 1, nullptr);
 
-  config.g_w = mFrameWidth;
-  config.g_h = mFrameHeight;
-  // TODO: Maybe we should have various aFrameRate bitrate pair for each devices?
-  // or for different platform
-
-  // rc_target_bitrate needs kbit/s
-  config.rc_target_bitrate = (mVideoBitrate != 0 ? mVideoBitrate : DEFAULT_BITRATE_BPS)/1000;
-
-  // Setting the time base of the codec
-  config.g_timebase.num = 1;
-  config.g_timebase.den = mTrackRate;
-
-  config.g_error_resilient = 0;
-
-  config.g_lag_in_frames = 0; // 0- no frame lagging
-
-  int32_t number_of_cores = PR_GetNumberOfProcessors();
-  if (mFrameWidth * mFrameHeight > 1280 * 960 && number_of_cores >= 6) {
-    config.g_threads = 3; // 3 threads for 1080p.
-  } else if (mFrameWidth * mFrameHeight > 640 * 480 && number_of_cores >= 3) {
-    config.g_threads = 2; // 2 threads for qHD/HD.
-  } else {
-    config.g_threads = 1; // 1 thread for VGA or less
-  }
-
-  // rate control settings
-  config.rc_dropframe_thresh = 0;
-  config.rc_end_usage = VPX_VBR;
-  config.g_pass = VPX_RC_ONE_PASS;
-  // ffmpeg doesn't currently support streams that use resize.
-  // Therefore, for safety, we should turn it off until it does.
-  config.rc_resize_allowed = 0;
-  config.rc_undershoot_pct = 100;
-  config.rc_overshoot_pct = 15;
-  config.rc_buf_initial_sz = 500;
-  config.rc_buf_optimal_sz = 600;
-  config.rc_buf_sz = 1000;
-
-  config.kf_mode = VPX_KF_AUTO;
-  // Ensure that we can output one I-frame per second.
-  config.kf_max_dist = 60;
-
   vpx_codec_flags_t flags = 0;
   flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
   if (vpx_codec_enc_init(mVPXContext, vpx_codec_vp8_cx(), &config, flags)) {
     return NS_ERROR_FAILURE;
   }
 
   vpx_codec_control(mVPXContext, VP8E_SET_STATIC_THRESHOLD, 1);
   vpx_codec_control(mVPXContext, VP8E_SET_CPUUSED, -6);
@@ -156,16 +121,126 @@ VP8TrackEncoder::Init(int32_t aWidth, in
                     VP8_ONE_TOKENPARTITION);
 
   mInitialized = true;
   mon.NotifyAll();
 
   return NS_OK;
 }
 
+nsresult
+VP8TrackEncoder::Reconfigure(int32_t aWidth, int32_t aHeight,
+                             int32_t aDisplayWidth, int32_t aDisplayHeight)
+{
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(aWidth > 0 && aHeight > 0 && aDisplayWidth > 0 &&
+             aDisplayHeight > 0);
+
+  ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+
+  mInitialized = false;
+  // Recreate image wrapper
+  vpx_img_free(mVPXImageWrapper);
+  vpx_img_wrap(mVPXImageWrapper, VPX_IMG_FMT_I420, aWidth, aHeight, 1, nullptr);
+  // Encoder configuration structure.
+  vpx_codec_enc_cfg_t config;
+  nsresult rv = SetConfigurationValues(aWidth, aHeight, aDisplayWidth, aDisplayHeight, config);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
+  // Set new configuration
+  if (vpx_codec_enc_config_set(mVPXContext.get(), &config) != VPX_CODEC_OK) {
+    VP8LOG(LogLevel::Error, "Failed to set new configuration");
+    return NS_ERROR_FAILURE;
+  }
+  mInitialized = true;
+  return NS_OK;
+}
+
+nsresult
+VP8TrackEncoder::SetConfigurationValues(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
+                                        int32_t aDisplayHeight, vpx_codec_enc_cfg_t& config)
+{
+   mFrameWidth = aWidth;
+   mFrameHeight = aHeight;
+   mDisplayWidth = aDisplayWidth;
+   mDisplayHeight = aDisplayHeight;
+
+   // Encoder configuration structure.
+   memset(&config, 0, sizeof(vpx_codec_enc_cfg_t));
+   if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config, 0)) {
+    VP8LOG(LogLevel::Error, "Failed to get default configuration");
+    return NS_ERROR_FAILURE;
+   }
+
+   config.g_w = mFrameWidth;
+   config.g_h = mFrameHeight;
+   // TODO: Maybe we should have various aFrameRate bitrate pair for each devices?
+   // or for different platform
+
+   // rc_target_bitrate needs kbit/s
+   config.rc_target_bitrate = (mVideoBitrate != 0 ? mVideoBitrate : DEFAULT_BITRATE_BPS)/1000;
+
+   // Setting the time base of the codec
+   config.g_timebase.num = 1;
+   config.g_timebase.den = mTrackRate;
+
+   config.g_error_resilient = 0;
+
+   config.g_lag_in_frames = 0; // 0- no frame lagging
+
+   int32_t number_of_cores = PR_GetNumberOfProcessors();
+   if (mFrameWidth * mFrameHeight > 1280 * 960 && number_of_cores >= 6) {
+     config.g_threads = 3; // 3 threads for 1080p.
+   } else if (mFrameWidth * mFrameHeight > 640 * 480 && number_of_cores >= 3) {
+     config.g_threads = 2; // 2 threads for qHD/HD.
+   } else {
+     config.g_threads = 1; // 1 thread for VGA or less
+   }
+
+   // rate control settings
+   config.rc_dropframe_thresh = 0;
+   config.rc_end_usage = VPX_CBR;
+   config.g_pass = VPX_RC_ONE_PASS;
+   // ffmpeg doesn't currently support streams that use resize.
+   // Therefore, for safety, we should turn it off until it does.
+   config.rc_resize_allowed = 0;
+   config.rc_undershoot_pct = 100;
+   config.rc_overshoot_pct = 15;
+   config.rc_buf_initial_sz = 500;
+   config.rc_buf_optimal_sz = 600;
+   config.rc_buf_sz = 1000;
+
+   config.kf_mode = VPX_KF_AUTO;
+   // Ensure that we can output one I-frame per second.
+   config.kf_max_dist = 60;
+
+   return NS_OK;
+}
+
+nsresult
+VP8TrackEncoder::ConfigureNewFrameSize(const VideoFrame& aFrame)
+{
+  gfx::IntSize intrinsicSize = aFrame.GetIntrinsicSize();
+  gfx::IntSize imgSize = aFrame.GetImage()->GetSize();
+  if (imgSize <= IntSize(mFrameWidth, mFrameHeight)) { // check buffer size instead
+    // If the new size is less than or equal to old,
+    // the existing encoder instance can continue.
+    if (NS_SUCCEEDED(Reconfigure(imgSize.width, imgSize.height, intrinsicSize.width,
+                                 intrinsicSize.height))) {
+      VP8LOG(LogLevel::Verbose, "ReConfigure VP8 encoder.");
+      return NS_OK;
+    }
+  }
+
+  // New frame size is larger; re-create the encoder.
+  Destroy();
+
+  return Init(imgSize.width, imgSize.height,
+              intrinsicSize.width, intrinsicSize.height);
+}
+
 already_AddRefed<TrackMetadataBase>
 VP8TrackEncoder::GetMetadata()
 {
   PROFILER_LABEL("VP8TrackEncoder", "GetMetadata",
     js::ProfileEntry::Category::OTHER);
   {
     // Wait if mEncoder is not initialized.
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
@@ -295,19 +370,20 @@ nsresult VP8TrackEncoder::PrepareRawFram
     }
     img = mMuteFrame;
   } else {
     img = aChunk.mFrame.GetImage();
   }
 
   if (img->GetSize() != IntSize(mFrameWidth, mFrameHeight)) {
     VP8LOG(LogLevel::Error,
-           "Dynamic resolution changes (was %dx%d, now %dx%d) are unsupported",
+           "Dynamic resolution changes (was %dx%d, now %dx%d).",
            mFrameWidth, mFrameHeight, img->GetSize().width, img->GetSize().height);
-    return NS_ERROR_FAILURE;
+    nsresult rv = ConfigureNewFrameSize(aChunk.mFrame);
+    NS_ENSURE_SUCCESS(rv, rv);
   }
 
   ImageFormat format = img->GetFormat();
   if (format == ImageFormat::PLANAR_YCBCR) {
     PlanarYCbCrImage* yuv = static_cast<PlanarYCbCrImage *>(img.get());
 
     MOZ_RELEASE_ASSERT(yuv);
     if (!yuv->IsValid()) {
@@ -330,17 +406,17 @@ nsresult VP8TrackEncoder::PrepareRawFram
   }
 
   // Not 420 planar, have to convert
   uint32_t yPlaneSize = mFrameWidth * mFrameHeight;
   uint32_t halfWidth = (mFrameWidth + 1) / 2;
   uint32_t halfHeight = (mFrameHeight + 1) / 2;
   uint32_t uvPlaneSize = halfWidth * halfHeight;
 
-  if (mI420Frame.IsEmpty()) {
+  if (mI420Frame.Length() != yPlaneSize + uvPlaneSize * 2) {
     mI420Frame.SetLength(yPlaneSize + uvPlaneSize * 2);
   }
 
   uint8_t *y = mI420Frame.Elements();
   uint8_t *cb = mI420Frame.Elements() + yPlaneSize;
   uint8_t *cr = mI420Frame.Elements() + yPlaneSize + uvPlaneSize;
 
   if (format == ImageFormat::PLANAR_YCBCR) {
@@ -562,16 +638,17 @@ VP8TrackEncoder::GetEncodedTrack(Encoded
       int flags = 0;
       if (nextEncodeOperation == ENCODE_I_FRAME) {
         VP8LOG(LogLevel::Warning, "MediaRecorder lagging behind. Encoding keyframe.");
         flags |= VPX_EFLAG_FORCE_KF;
       }
       if (vpx_codec_encode(mVPXContext, mVPXImageWrapper, mEncodedTimestamp,
                            (unsigned long)chunk.GetDuration(), flags,
                            VPX_DL_REALTIME)) {
+        VP8LOG(LogLevel::Error, "vpx_codec_encode failed to encode the frame.");
         return NS_ERROR_FAILURE;
       }
       // Get the encoded data from VP8 encoder.
       rv = GetEncodedPartitions(aData);
       NS_ENSURE_SUCCESS(rv, NS_ERROR_FAILURE);
     } else {
       // SKIP_FRAME
       // Extend the duration of the last encoded data in aData
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -47,16 +47,30 @@ private:
   // Get the encoded data from encoder to aData.
   // Return value: false if the vpx_codec_get_cx_data returns null
   //               for EOS detection.
   nsresult GetEncodedPartitions(EncodedFrameContainer& aData);
 
   // Prepare the input data to the mVPXImageWrapper for encoding.
   nsresult PrepareRawFrame(VideoChunk &aChunk);
 
+  // Re-configures an existing encoder with a new frame size.
+  nsresult Reconfigure(int32_t aWidth, int32_t aHeight,
+                       int32_t aDisplayWidth, int32_t aDisplayHeight);
+
+  // Destroys the context and image wrapper. Does not de-allocate the structs.
+  void Destroy();
+
+  // Helper method to set the values on a VPX configuration.
+  nsresult SetConfigurationValues(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
+                                  int32_t aDisplayHeight, vpx_codec_enc_cfg_t& config);
+
+  // Configure encoder when new frame size (resolution).
+  nsresult ConfigureNewFrameSize(const VideoFrame& aFrame);
+
   // Encoded timestamp.
   StreamTime mEncodedTimestamp;
 
   // Total duration in mTrackRate extracted by GetEncodedPartitions().
   CheckedInt64 mExtractedDuration;
 
   // Total duration in microseconds extracted by GetEncodedPartitions().
   CheckedInt64 mExtractedDurationUs;