Bug 1432779 - P13. Fix input mixing and clarify frames vs samples terminology. r?padenot draft
authorJean-Yves Avenard <jyavenard@mozilla.com>
Thu, 15 Mar 2018 17:39:54 +0100
changeset 768210 a996321bbc2b4aac49dfa539207efc5a50f8f5f7
parent 768209 cdd5ee6e882c512f4d30ef296124c9fc38b65ff3
child 768211 322373f293a7a7ad52574cd83cf72d61540d3583
push id102819
push userbmo:jyavenard@mozilla.com
push dateThu, 15 Mar 2018 19:46:59 +0000
reviewerspadenot
bugs1432779
milestone61.0a1
Bug 1432779 - P13. Fix input mixing and clarify frames vs samples terminology. r?padenot MozReview-Commit-ID: 7M9xJB7HDMB
media/libcubeb/src/cubeb_wasapi.cpp
--- a/media/libcubeb/src/cubeb_wasapi.cpp
+++ b/media/libcubeb/src/cubeb_wasapi.cpp
@@ -268,19 +268,20 @@ struct cubeb_stream {
      client, frames_written, mix_params, total_frames_written, prev_position. */
   owned_critical_section stream_reset_lock;
   /* Maximum number of frames that can be passed down in a callback. */
   uint32_t input_buffer_frame_count = 0;
   /* Maximum number of frames that can be requested in a callback. */
   uint32_t output_buffer_frame_count = 0;
   /* Resampler instance. Resampling will only happen if necessary. */
   std::unique_ptr<cubeb_resampler, decltype(&cubeb_resampler_destroy)> resampler = { nullptr, cubeb_resampler_destroy };
-  /* Mixer interface */
-  std::unique_ptr<cubeb_mixer, decltype(&cubeb_mixer_destroy)> mixer = { nullptr, cubeb_mixer_destroy };
-  /* A buffer for up/down mixing multi-channel audio. */
+  /* Mixer interfaces */
+  std::unique_ptr<cubeb_mixer, decltype(&cubeb_mixer_destroy)> output_mixer = { nullptr, cubeb_mixer_destroy };
+  std::unique_ptr<cubeb_mixer, decltype(&cubeb_mixer_destroy)> input_mixer = { nullptr, cubeb_mixer_destroy };
+  /* A buffer for up/down mixing multi-channel audio output. */
   std::vector<BYTE> mix_buffer;
   /* WASAPI input works in "packets". We re-linearize the audio packets
    * into this buffer before handing it to the resampler. */
   std::unique_ptr<auto_array_wrapper> linear_input_buffer;
   /* Bytes per sample. This multiplied by the number of channels is the number
    * of bytes per frame. */
   size_t bytes_per_sample = 0;
   /* WAVEFORMATEXTENSIBLE sub-format: either PCM or float. */
@@ -487,17 +488,17 @@ long
 refill(cubeb_stream * stm, void * input_buffer, long input_frames_count,
        void * output_buffer, long output_frames_needed)
 {
   XASSERT(!stm->draining);
   /* If we need to upmix after resampling, resample into the mix buffer to
      avoid a copy. Avoid exposing output if it is a dummy stream. */
   void * dest = nullptr;
   if (has_output(stm) && !stm->has_dummy_output) {
-    if (stm->mixer) {
+    if (stm->output_mixer) {
       dest = stm->mix_buffer.data();
     } else {
       dest = output_buffer;
     }
   }
 
   long out_frames = cubeb_resampler_fill(stm->resampler.get(),
                                          input_buffer,
@@ -518,23 +519,23 @@ refill(cubeb_stream * stm, void * input_
     stm->draining = true;
   }
 
   /* If this is not true, there will be glitches.
      It is alright to have produced less frames if we are draining, though. */
   XASSERT(out_frames == output_frames_needed || stm->draining || !has_output(stm) || stm->has_dummy_output);
 
   // We don't bother mixing dummy output as it will be silenced, otherwise mix output if needed
-  if (!stm->has_dummy_output && has_output(stm) && stm->mixer) {
+  if (!stm->has_dummy_output && has_output(stm) && stm->output_mixer) {
     XASSERT(dest == stm->mix_buffer.data());
     unsigned long dest_size =
       out_frames * stm->output_stream_params.channels * stm->bytes_per_sample;
     XASSERT(dest_size <= stm->mix_buffer.size());
     unsigned long output_buffer_size = out_frames * stm->output_mix_params.channels * stm->bytes_per_sample;
-    int ret = cubeb_mixer_mix(stm->mixer.get(),
+    int ret = cubeb_mixer_mix(stm->output_mixer.get(),
                               out_frames,
                               dest,
                               dest_size,
                               output_buffer,
                               output_buffer_size);
     if (ret < 0) {
       LOG("Error remixing content (%d)", ret);
     }
@@ -575,55 +576,76 @@ bool get_input_buffer(cubeb_stream * stm
       return true;
     }
 
     if (FAILED(hr)) {
       LOG("cannot get next packet size: %lx", hr);
       return false;
     }
 
-    UINT32 packet_size;
+    UINT32 frames;
     hr = stm->capture_client->GetBuffer(&input_packet,
-                                        &packet_size,
+                                        &frames,
                                         &flags,
                                         &dev_pos,
                                         NULL);
     if (FAILED(hr)) {
       LOG("GetBuffer failed for capture: %lx", hr);
       return false;
     }
-    XASSERT(packet_size == next);
+    XASSERT(frames == next);
+
+    UINT32 input_stream_samples = frames * stm->input_stream_params.channels;
     // We do not explicitly handle the AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY
     // flag. There a two primary (non exhaustive) scenarios we anticipate this
     // flag being set in:
     //   - The first GetBuffer after Start has this flag undefined. In this
     //     case the flag may be set but is meaningless and can be ignored.
     //   - If a glitch is introduced into the input. This should not happen
     //     for event based inputs, and should be mitigated by using a dummy
     //     stream to drive input in the case of input only loopback. Without
     //     a dummy output, input only loopback would glitch on silence. However,
     //     the dummy input should push silence to the loopback and prevent
     //     discontinuities. See https://blogs.msdn.microsoft.com/matthew_van_eerde/2008/12/16/sample-wasapi-loopback-capture-record-what-you-hear/
     // As the first scenario can be ignored, and we anticipate the second
     // scenario is mitigated, we ignore the flag.
     // For more info: https://msdn.microsoft.com/en-us/library/windows/desktop/dd370859(v=vs.85).aspx,
     // https://msdn.microsoft.com/en-us/library/windows/desktop/dd371458(v=vs.85).aspx
     if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
-      LOG("insert silence: ps=%u", packet_size);
-      stm->linear_input_buffer->push_silence(packet_size * stm->input_stream_params.channels);
+      LOG("insert silence: ps=%u", frames);
+      stm->linear_input_buffer->push_silence(input_stream_samples);
     } else {
-      stm->linear_input_buffer->push(input_packet,
-                                     packet_size * stm->input_stream_params.channels);
+      if (stm->input_mixer) {
+        bool ok = stm->linear_input_buffer->reserve(
+          stm->linear_input_buffer->length() + input_stream_samples);
+        XASSERT(ok);
+        unsigned long input_packet_size =
+          frames * stm->input_mix_params.channels *
+          cubeb_sample_size(stm->input_mix_params.format);
+        unsigned long linear_input_buffer_size =
+          input_stream_samples * cubeb_sample_size(stm->input_stream_params.format);
+        cubeb_mixer_mix(stm->input_mixer.get(),
+                        frames,
+                        input_packet,
+                        input_packet_size,
+                        stm->linear_input_buffer->end(),
+                        linear_input_buffer_size);
+        stm->linear_input_buffer->set_length(
+          stm->linear_input_buffer->length() + input_stream_samples);
+      } else {
+        stm->linear_input_buffer->push(
+          input_packet, input_stream_samples);
+      }
     }
-    hr = stm->capture_client->ReleaseBuffer(packet_size);
+    hr = stm->capture_client->ReleaseBuffer(frames);
     if (FAILED(hr)) {
       LOG("FAILED to release intput buffer");
       return false;
     }
-    offset += packet_size;
+    offset += input_stream_samples;
   }
 
   XASSERT(stm->linear_input_buffer->length() >= offset);
 
   return true;
 }
 
 /* Get an output buffer from the render_client. It has to be released before
@@ -1739,27 +1761,44 @@ int setup_wasapi_stream(cubeb_stream * s
   if (has_input(stm) && has_output(stm)) {
     stm->refill_callback = refill_callback_duplex;
   } else if (has_input(stm)) {
     stm->refill_callback = refill_callback_input;
   } else if (has_output(stm)) {
     stm->refill_callback = refill_callback_output;
   }
 
-  // Create mixer.
+  // Create input mixer.
+  if (has_input(stm) &&
+      ((stm->input_mix_params.layout != CUBEB_LAYOUT_UNDEFINED &&
+        stm->input_mix_params.layout != stm->input_stream_params.layout) ||
+       (stm->input_mix_params.channels != stm->input_stream_params.channels))) {
+    if (stm->input_mix_params.layout == CUBEB_LAYOUT_UNDEFINED) {
+      LOG("Input stream using undefined layout! Any mixing may be "
+          "unpredictable!\n");
+    }
+    stm->input_mixer.reset(cubeb_mixer_create(stm->input_stream_params.format,
+                                              stm->input_mix_params.channels,
+                                              stm->input_mix_params.layout,
+                                              stm->input_stream_params.channels,
+                                              stm->input_stream_params.layout));
+    assert(stm->input_mixer);
+  }
+
+  // Create output mixer.
   if (has_output(stm) && stm->output_mix_params.layout != stm->output_stream_params.layout) {
     if (stm->output_mix_params.layout == CUBEB_LAYOUT_UNDEFINED) {
-      LOG("Stream using undefined layout! Any mixing may be unpredictable!\n");
+      LOG("Output stream using undefined layout! Any mixing may be unpredictable!\n");
     }
-    stm->mixer.reset(cubeb_mixer_create(stm->output_stream_params.format,
-                                        stm->output_stream_params.channels,
-                                        stm->output_stream_params.layout,
-                                        stm->output_mix_params.channels,
-                                        stm->output_mix_params.layout));
-    assert(stm->mixer);
+    stm->output_mixer.reset(cubeb_mixer_create(stm->output_stream_params.format,
+                                               stm->output_stream_params.channels,
+                                               stm->output_stream_params.layout,
+                                               stm->output_mix_params.channels,
+                                               stm->output_mix_params.layout));
+    assert(stm->output_mixer);
     // Input is up/down mixed when depacketized in get_input_buffer.
     stm->mix_buffer.resize(
       frames_to_bytes_before_mix(stm, stm->output_buffer_frame_count));
   }
 
   return CUBEB_OK;
 }
 
@@ -1877,17 +1916,18 @@ void close_wasapi_stream(cubeb_stream * 
 
   stm->audio_stream_volume = nullptr;
 
   stm->audio_clock = nullptr;
   stm->total_frames_written += static_cast<UINT64>(round(stm->frames_written * stream_to_mix_samplerate_ratio(stm->output_stream_params, stm->output_mix_params)));
   stm->frames_written = 0;
 
   stm->resampler.reset();
-  stm->mixer.reset();
+  stm->output_mixer.reset();
+  stm->input_mixer.reset();
   stm->mix_buffer.clear();
 }
 
 void wasapi_stream_destroy(cubeb_stream * stm)
 {
   XASSERT(stm);
   LOG("Stream destroy (%p)", stm);