Bug 1221587: patch 5 - Base update of the MSG API for full-duplex r?padenot draft
authorRandell Jesup <rjesup@jesup.org>
Sun, 10 Jan 2016 21:30:59 -0500
changeset 321193 24f31e31725cfa875ad6c46e190602965ce7d888
parent 321192 375da87ac5f8be8eb6f99550be13a014b74080a7
child 321194 566b432289c3c3e37dc43972e5998cefd6d81e5b
push id9349
push userrjesup@wgate.com
push dateWed, 13 Jan 2016 06:48:48 +0000
reviewerspadenot
bugs1221587
milestone46.0a1
Bug 1221587: patch 5 - Base update of the MSG API for full-duplex r?padenot
dom/media/GraphDriver.cpp
dom/media/GraphDriver.h
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/MediaStreamGraphImpl.h
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@@ -881,16 +881,33 @@ AudioCallbackDriver::DataCallback(AudioD
     stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
   } else {
     NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
     stillProcessing = true;
   }
 
   mBuffer.BufferFilled();
 
+  // Callback any observers for the AEC speaker data.  Note that one
+  // (maybe) of these will be full-duplex, the others will get their input
+  // data off separate cubeb callbacks.  Take care with how stuff is
+  // removed/added to this list and TSAN issues, but input and output will
+  // use separate callback methods.
+  mGraphImpl->NotifySpeakerData(aOutputBuffer, static_cast<size_t>(aFrames),
+                                ChannelCount);
+
+  // Process mic data if any/needed -- after inserting far-end data for AEC!
+  if (aInputBuffer) {
+    if (mAudioInput) { // for this specific input-only or full-duplex stream
+      mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer,
+                                   static_cast<size_t>(aFrames),
+                                   ChannelCount);
+    }
+  }
+
   bool switching = false;
   {
     MonitorAutoLock mon(mGraphImpl->GetMonitor());
     switching = !!NextDriver();
   }
 
   if (switching && stillProcessing) {
     // If the audio stream has not been started by the previous driver or
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@@ -191,16 +191,26 @@ public:
   void EnsureNextIterationLocked();
 
   MediaStreamGraphImpl* GraphImpl() {
     return mGraphImpl;
   }
 
   virtual bool OnThread() = 0;
 
+  // XXX Thread-safety! Do these via commands to avoid TSAN issues
+  // and crashes!!!
+  virtual void SetInputListener(MediaStreamListener *aListener) {
+    mAudioInput = aListener;
+  }
+  // XXX do we need the param?  probably no
+  virtual void RemoveInputListener(MediaStreamListener *aListener) {
+    mAudioInput = nullptr;
+  }
+
 protected:
   GraphTime StateComputedTime() const;
 
   // Time of the start of this graph iteration. This must be accessed while
   // having the monitor.
   GraphTime mIterationStart;
   // Time of the end of this graph iteration. This must be accessed while having
   // the monitor.
@@ -221,16 +231,19 @@ protected:
     WAITSTATE_WAITING_INDEFINITELY,
     // Something has signaled RunThread() to wake up immediately,
     // but it hasn't done so yet
     WAITSTATE_WAKING_UP
   };
   // This must be access with the monitor.
   WaitState mWaitState;
 
+  // Callback for mic data, if any
+  RefPtr<MediaStreamListener> mAudioInput;
+
   // This is used on the main thread (during initialization), and the graph
   // thread. No monitor needed because we know the graph thread does not run
   // during the initialization.
   TimeStamp mCurrentTimeStamp;
   // This is non-null only when this driver has recently switched from an other
   // driver, and has not cleaned it up yet (for example because the audio stream
   // is currently calling the callback during initialization).
   //
@@ -483,16 +496,18 @@ private:
    * This is written on the previous driver's thread (if switching) or main
    * thread (if this driver is the first one).
    * This is read on previous driver's thread (during callbacks from
    * cubeb_stream_init) and the audio thread (when switching away from this
    * driver back to a SystemClockDriver).
    * This is synchronized by the Graph's monitor.
    * */
   bool mStarted;
+  /* Listener for mic input, if any. */
+  RefPtr<MediaStreamListener> mAudioInput;
 
   struct AutoInCallback
   {
     explicit AutoInCallback(AudioCallbackDriver* aDriver);
     ~AutoInCallback();
     AudioCallbackDriver* mDriver;
   };
 
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -25,16 +25,17 @@
 #include "mozilla/dom/AudioContextBinding.h"
 #include <algorithm>
 #include "DOMMediaStream.h"
 #include "GeckoProfiler.h"
 #include "mozilla/unused.h"
 #ifdef MOZ_WEBRTC
 #include "AudioOutputObserver.h"
 #endif
+#include "mtransport/runnable_utils.h"
 
 #include "webaudio/blink/HRTFDatabaseLoader.h"
 
 using namespace mozilla::layers;
 using namespace mozilla::dom;
 using namespace mozilla::gfx;
 
 namespace mozilla {
@@ -917,16 +918,95 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
 
   // If the stream has finished and the timestamps of all frames have expired
   // then no more updates are required.
   if (aStream->mFinished && !haveMultipleImages) {
     aStream->mLastPlayedVideoFrame.SetNull();
   }
 }
 
+void
+MediaStreamGraphImpl::OpenAudioInputImpl(char *aName, MediaStreamListener *aListener)
+{
+  if (CurrentDriver()->AsAudioCallbackDriver()) {
+    CurrentDriver()->SetInputListener(aListener);
+  } else {
+    // XXX Switch to callback driver
+  }
+  mAudioInputs.AppendElement(aListener); // always monitor speaker data
+}
+
+nsresult
+MediaStreamGraphImpl::OpenAudioInput(char *aName, MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::OpenAudioInput,
+                                         aName, aListener)); // XXX Fix! string need to copied
+    return NS_OK;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, char *aName, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mName(aName), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->OpenAudioInputImpl(mName, mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    char *mName; // XXX needs to copy
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aName, aListener));
+  return NS_OK;
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInputImpl(MediaStreamListener *aListener)
+{
+  CurrentDriver()->RemoveInputListener(aListener);
+  mAudioInputs.RemoveElement(aListener);
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInput(MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::CloseAudioInput,
+                                         aListener));
+    return;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->CloseAudioInputImpl(mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aListener));
+}
+
+
+// All AudioInput listeners get the same speaker data (at least for now).
+void
+MediaStreamGraph::NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                                    uint32_t aChannels)
+{
+  for (auto& listener : mAudioInputs) {
+    listener->NotifySpeakerData(this, aBuffer, aFrames, aChannels);
+  }
+}
+
 bool
 MediaStreamGraphImpl::ShouldUpdateMainThread()
 {
   if (mRealtime) {
     return true;
   }
 
   TimeStamp now = TimeStamp::Now();
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -176,16 +176,33 @@ public:
                                         TrackID aInputTrackID = TRACK_INVALID) {}
 
   /**
    * Notify that all new tracks this iteration have been created.
    * This is to ensure that tracks added atomically to MediaStreamGraph
    * are also notified of atomically to MediaStreamListeners.
    */
   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
+
+  /* These are for cubeb audio input & output streams: */
+  /**
+   * Output data to speakers, for use as the "far-end" data for echo
+   * cancellation.  This is not guaranteed to be in any particular size
+   * chunks.
+   */
+  virtual void NotifySpeakerData(MediaStreamGraph* aGraph,
+                                 AudioDataValue* aBuffer, size_t aFrames,
+                                 uint32_t aChannels) {}
+  /**
+   * Input data from a microphone (or other audio source.  This is not
+   * guaranteed to be in any particular size chunks.
+   */
+  virtual void NotifyInputData(MediaStreamGraph* aGraph,
+                               AudioDataValue* aBuffer, size_t aFrames,
+                               uint32_t aChannels) {}
 };
 
 /**
  * This is a base class for media graph thread listener direct callbacks
  * from within AppendToTrack().  Note that your regular listener will
  * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
  * you must be careful to ignore them if AddDirectListener was successful.
  */
@@ -1170,16 +1187,21 @@ public:
   };
   // Main thread only
   static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
                                        dom::AudioChannel aChannel);
   static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
 
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) {
+    return NS_ERROR_FAILURE;
+  }
+  virtual void CloseAudioInput(MediaStreamListener *aListener) {}
+
   // Control API.
   /**
    * Create a stream that a media decoder (or some other source of
    * media data, such as a camera) can write to.
    */
   SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
   /**
    * Create a stream that will form the union of the tracks of its input
@@ -1249,16 +1271,23 @@ public:
   TrackRate GraphRate() const { return mSampleRate; }
 
   void RegisterCaptureStreamForWindow(uint64_t aWindowId,
                                       ProcessedMediaStream* aCaptureStream);
   void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
   already_AddRefed<MediaInputPort> ConnectToCaptureStream(
     uint64_t aWindowId, MediaStream* aMediaStream);
 
+  /**
+   * Data going to the speakers from the GraphDriver's DataCallback
+   * to notify any listeners (for echo cancellation).
+   */
+  void NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                         uint32_t aChannels);
+
 protected:
   explicit MediaStreamGraph(TrackRate aSampleRate)
     : mSampleRate(aSampleRate)
   {
     MOZ_COUNT_CTOR(MediaStreamGraph);
   }
   virtual ~MediaStreamGraph()
   {
@@ -1269,13 +1298,15 @@ protected:
   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
 
   /**
    * Sample rate at which this graph runs. For real time graphs, this is
    * the rate of the audio mixer. For offline graphs, this is the rate specified
    * at construction.
    */
   TrackRate mSampleRate;
+
+  nsTArray<RefPtr<MediaStreamListener>> mAudioInputs;
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
--- a/dom/media/MediaStreamGraphImpl.h
+++ b/dom/media/MediaStreamGraphImpl.h
@@ -345,16 +345,21 @@ public:
    * Set the correct current video frame for stream aStream.
    */
   void PlayVideo(MediaStream* aStream);
   /**
    * No more data will be forthcoming for aStream. The stream will end
    * at the current buffer end point. The StreamBuffer's tracks must be
    * explicitly set to finished by the caller.
    */
+  void OpenAudioInputImpl(char *aName, MediaStreamListener *aListener);
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) override;
+  void CloseAudioInputImpl(MediaStreamListener *aListener);
+  virtual void CloseAudioInput(MediaStreamListener *aListener) override;
+
   void FinishStream(MediaStream* aStream);
   /**
    * Compute how much stream data we would like to buffer for aStream.
    */
   StreamTime GetDesiredBufferEnd(MediaStream* aStream);
   /**
    * Returns true when there are no active streams.
    */