Bug 1404977 - Part 8 - Tell the MSG the MediaEngineAudioSource are now independant and that we can have multiple of them, cleanup the MSG-side API for managing them. r?pehrsons draft
authorPaul Adenot <paul@paul.cx>
Mon, 30 Apr 2018 16:01:38 +0200
changeset 794148 005538dcf876aad4c2d4c84a86e02d50195da41f
parent 794147 79ebf1b2bd24b8da8da3bd2fbee5dc196637b18a
child 794149 c2c20761199f18e7ed5fb52f14a76687c71213f2
push id109576
push userachronop@gmail.com
push dateFri, 11 May 2018 11:11:31 +0000
reviewerspehrsons
bugs1404977
milestone62.0a1
Bug 1404977 - Part 8 - Tell the MSG the MediaEngineAudioSource are now independant and that we can have multiple of them, cleanup the MSG-side API for managing them. r?pehrsons The MSG now can feed microphone data to all its input listener. This paves the way for multiple input device, if we feel it's needed at some point, but does not implement it. The method for adding/removing inputs are also cleaned up. MozReview-Commit-ID: 24mjuIeozor
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/MediaStreamGraphImpl.h
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -791,173 +791,220 @@ MediaStreamGraphImpl::PlayAudio(MediaStr
                                      mMixer,
                                      AudioChannelCount(),
                                      mSampleRate);
   }
   return ticksWritten;
 }
 
 void
-MediaStreamGraphImpl::OpenAudioInputImpl(int aID,
+MediaStreamGraphImpl::OpenAudioInputImpl(CubebUtils::AudioDeviceID aID,
                                          AudioDataListener *aListener)
 {
   MOZ_ASSERT(OnGraphThreadOrNotRunning());
-  // Bug 1238038 Need support for multiple mics at once
-  if (mInputDeviceUsers.Count() > 0 &&
-      !mInputDeviceUsers.Get(aListener, nullptr)) {
-    NS_ASSERTION(false, "Input from multiple mics not yet supported; bug 1238038");
-    // Need to support separate input-only AudioCallback drivers; they'll
-    // call us back on "other" threads.  We will need to echo-cancel them, though.
+  // Only allow one device per MSG (hence, per document), but allow opening a
+  // device multiple times
+  nsTArray<RefPtr<AudioDataListener>>& listeners = mInputDeviceUsers.GetOrInsert(aID);
+  // We don't support opening multiple input device in a graph for now.
+  if (listeners.IsEmpty() && mInputDeviceUsers.Count() > 1) {
+    listeners.RemoveElement(aID);
     return;
   }
-  mInputWanted = true;
-
-  // Add to count of users for this ID.
-  // XXX Since we can't rely on IDs staying valid (ugh), use the listener as
-  // a stand-in for the ID.  Fix as part of support for multiple-captures
-  // (Bug 1238038)
-  uint32_t count = 0;
-  mInputDeviceUsers.Get(aListener, &count); // ok if this fails
-  count++;
-  mInputDeviceUsers.Put(aListener, count); // creates a new entry in the hash if needed
-
-  if (count == 1) { // first open for this listener
-    // aID is a cubeb_devid, and we assume that opaque ptr is valid until
-    // we close cubeb.
+
+  MOZ_ASSERT(!listeners.Contains(aListener), "Don't add a listener twice.");
+
+  listeners.AppendElement(aListener);
+
+  if (listeners.Length() == 1) { // first open for this device
     mInputDeviceID = aID;
-    mAudioInputs.AppendElement(aListener); // always monitor speaker data
-
     // Switch Drivers since we're adding input (to input-only or full-duplex)
     MonitorAutoLock mon(mMonitor);
     if (LifecycleStateRef() == LIFECYCLE_RUNNING) {
-      AudioCallbackDriver* driver = new AudioCallbackDriver(this);
-      driver->SetMicrophoneActive(true);
+      AudioCallbackDriver* driver = new AudioCallbackDriver(this, AudioInputChannelCount());
       LOG(
         LogLevel::Debug,
         ("OpenAudioInput: starting new AudioCallbackDriver(input) %p", driver));
       LOG(
         LogLevel::Debug,
         ("OpenAudioInput: starting new AudioCallbackDriver(input) %p", driver));
       driver->SetInputListener(aListener);
       CurrentDriver()->SwitchAtNextIteration(driver);
    } else {
      LOG(LogLevel::Error, ("OpenAudioInput in shutdown!"));
-     LOG(LogLevel::Debug, ("OpenAudioInput in shutdown!"));
-     NS_ASSERTION(false, "Can't open cubeb inputs in shutdown");
+     MOZ_ASSERT(false, "Can't open cubeb inputs in shutdown");
     }
   }
 }
 
 nsresult
-MediaStreamGraphImpl::OpenAudioInput(int aID,
+MediaStreamGraphImpl::OpenAudioInput(CubebUtils::AudioDeviceID aID,
                                      AudioDataListener *aListener)
 {
   // So, so, so annoying.  Can't AppendMessage except on Mainthread
   if (!NS_IsMainThread()) {
     RefPtr<nsIRunnable> runnable =
       WrapRunnable(this,
                    &MediaStreamGraphImpl::OpenAudioInput,
                    aID,
                    RefPtr<AudioDataListener>(aListener));
     mAbstractMainThread->Dispatch(runnable.forget());
     return NS_OK;
   }
   class Message : public ControlMessage {
   public:
-    Message(MediaStreamGraphImpl *aGraph, int aID,
+    Message(MediaStreamGraphImpl *aGraph, CubebUtils::AudioDeviceID aID,
             AudioDataListener *aListener) :
       ControlMessage(nullptr), mGraph(aGraph), mID(aID), mListener(aListener) {}
     void Run() override
     {
       mGraph->OpenAudioInputImpl(mID, mListener);
     }
     MediaStreamGraphImpl *mGraph;
-    int mID;
+    CubebUtils::AudioDeviceID mID;
     RefPtr<AudioDataListener> mListener;
   };
   // XXX Check not destroyed!
   this->AppendMessage(MakeUnique<Message>(this, aID, aListener));
   return NS_OK;
 }
 
 void
-MediaStreamGraphImpl::CloseAudioInputImpl(AudioDataListener *aListener)
+MediaStreamGraphImpl::CloseAudioInputImpl(CubebUtils::AudioDeviceID aID, AudioDataListener* aListener)
 {
   MOZ_ASSERT(OnGraphThreadOrNotRunning());
-  uint32_t count;
-  DebugOnly<bool> result = mInputDeviceUsers.Get(aListener, &count);
-  MOZ_ASSERT(result);
-  if (--count > 0) {
-    mInputDeviceUsers.Put(aListener, count);
-    return; // still in use
+  // It is possible to not know the ID here, find it first.
+  if (aID == nullptr) {
+    for (auto iter = mInputDeviceUsers.Iter(); !iter.Done(); iter.Next()) {
+      if (iter.Data().Contains(aListener)) {
+        aID = iter.Key();
+      }
+    }
+    MOZ_ASSERT(aID != nullptr, "Closing an audio input that was not opened.");
   }
-  mInputDeviceUsers.Remove(aListener);
-  mInputDeviceID = -1;
-  mInputWanted = false;
-  AudioCallbackDriver *driver = CurrentDriver()->AsAudioCallbackDriver();
-  if (driver) {
-    driver->RemoveInputListener(aListener);
+
+  nsTArray<RefPtr<AudioDataListener>>* listeners = mInputDeviceUsers.GetValue(aID);
+
+  MOZ_ASSERT(listeners);
+  DebugOnly<bool> wasPresent = listeners->RemoveElement(aListener);
+  MOZ_ASSERT(wasPresent);
+  // Check whether there is still a consumer for this audio input device
+  if (!listeners->IsEmpty()) {
+    return;
   }
-  mAudioInputs.RemoveElement(aListener);
+
+  mInputDeviceID = nullptr; // reset to default
+  mInputDeviceUsers.Remove(aID);
 
   // Switch Drivers since we're adding or removing an input (to nothing/system or output only)
   bool audioTrackPresent = AudioTrackPresent();
 
   MonitorAutoLock mon(mMonitor);
   if (LifecycleStateRef() == LIFECYCLE_RUNNING) {
     GraphDriver* driver;
     if (audioTrackPresent) {
       // We still have audio output
-      LOG(LogLevel::Debug, ("CloseInput: output present (AudioCallback)"));
-
-      driver = new AudioCallbackDriver(this);
+      LOG(LogLevel::Debug, ("%p: CloseInput: output present (AudioCallback)", this));
+
+      driver = new AudioCallbackDriver(this, AudioInputChannelCount());
       CurrentDriver()->SwitchAtNextIteration(driver);
     } else if (CurrentDriver()->AsAudioCallbackDriver()) {
       LOG(LogLevel::Debug,
           ("CloseInput: no output present (SystemClockCallback)"));
 
       driver = new SystemClockDriver(this);
       CurrentDriver()->SwitchAtNextIteration(driver);
     } // else SystemClockDriver->SystemClockDriver, no switch
   }
 }
 
 void
-MediaStreamGraphImpl::CloseAudioInput(AudioDataListener *aListener)
+MediaStreamGraphImpl::CloseAudioInput(CubebUtils::AudioDeviceID aID, AudioDataListener* aListener)
 {
   // So, so, so annoying.  Can't AppendMessage except on Mainthread
   if (!NS_IsMainThread()) {
     RefPtr<nsIRunnable> runnable =
       WrapRunnable(this,
                    &MediaStreamGraphImpl::CloseAudioInput,
+                   aID,
                    RefPtr<AudioDataListener>(aListener));
     mAbstractMainThread->Dispatch(runnable.forget());
     return;
   }
   class Message : public ControlMessage {
   public:
-    Message(MediaStreamGraphImpl *aGraph, AudioDataListener *aListener) :
-      ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
+    Message(MediaStreamGraphImpl *aGraph, CubebUtils::AudioDeviceID aID, AudioDataListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mID(aID), mListener(aListener) {}
     void Run() override
     {
-      mGraph->CloseAudioInputImpl(mListener);
+      mGraph->CloseAudioInputImpl(mID, mListener);
     }
     MediaStreamGraphImpl *mGraph;
+    CubebUtils::AudioDeviceID mID;
     RefPtr<AudioDataListener> mListener;
   };
-  this->AppendMessage(MakeUnique<Message>(this, aListener));
+  this->AppendMessage(MakeUnique<Message>(this, aID, aListener));
 }
 
 // All AudioInput listeners get the same speaker data (at least for now).
 void
-MediaStreamGraph::NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
-                                   TrackRate aRate, uint32_t aChannels)
+MediaStreamGraphImpl::NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
+                                       TrackRate aRate, uint32_t aChannels)
+{
+  if (!mInputDeviceID) {
+    return;
+  }
+  // When/if we decide to support multiple input device per graph, this needs
+  // to loop over them.
+  nsTArray<RefPtr<AudioDataListener>>* listeners = mInputDeviceUsers.GetValue(mInputDeviceID);
+  MOZ_ASSERT(listeners);
+  for (auto& listener : *listeners) {
+    listener->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
+  }
+}
+
+void
+MediaStreamGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
+                                      TrackRate aRate, uint32_t aChannels)
 {
-  for (auto& listener : mAudioInputs) {
-    listener->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels);
+#ifdef DEBUG
+  MonitorAutoLock lock(mMonitor);
+  // Either we have an audio input device, or we just removed the audio input
+  // this iteration, and we're switching back to an output-only driver next
+  // iteration.
+  MOZ_ASSERT(mInputDeviceID || CurrentDriver()->Switching());
+#endif
+  nsTArray<RefPtr<AudioDataListener>>* listeners = mInputDeviceUsers.GetValue(mInputDeviceID);
+  MOZ_ASSERT(listeners);
+  for (auto& listener : *listeners) {
+    listener->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels);
+  }
+}
+
+void MediaStreamGraphImpl::DeviceChanged()
+{
+  if (!mInputDeviceID) {
+    return;
+  }
+  nsTArray<RefPtr<AudioDataListener>>* listeners = mInputDeviceUsers.GetValue(mInputDeviceID);
+  for (auto& listener : *listeners) {
+    listener->DeviceChanged();
+  }
+}
+
+void MediaStreamGraphImpl::ReevaluateInputDevice()
+{
+  MOZ_ASSERT(OnGraphThread());
+  AudioCallbackDriver* audioCallbackDriver = CurrentDriver()->AsAudioCallbackDriver();
+  MOZ_ASSERT(audioCallbackDriver);
+  if (audioCallbackDriver->InputChannelCount() != AudioInputChannelCount()) {
+    AudioCallbackDriver* newDriver = new AudioCallbackDriver(this, AudioInputChannelCount());
+    {
+      MonitorAutoLock lock(mMonitor);
+      CurrentDriver()->SwitchAtNextIteration(newDriver);
+    }
   }
 }
 
 bool
 MediaStreamGraph::OnGraphThreadOrNotRunning() const
 {
   // either we're on the right thread (and calling CurrentDriver() is safe),
   // or we're going to fail the assert anyway, so don't cross-check
@@ -2693,40 +2740,42 @@ SourceMediaStream::SourceMediaStream()
   , mUpdateKnownTracksTime(0)
   , mPullEnabled(false)
   , mFinishPending(false)
   , mNeedsMixing(false)
 {
 }
 
 nsresult
-SourceMediaStream::OpenAudioInput(int aID,
+SourceMediaStream::OpenAudioInput(CubebUtils::AudioDeviceID aID,
                                   AudioDataListener *aListener)
 {
   if (GraphImpl()) {
     mInputListener = aListener;
     return GraphImpl()->OpenAudioInput(aID, aListener);
   }
   return NS_ERROR_FAILURE;
 }
 
 void
-SourceMediaStream::CloseAudioInput()
+SourceMediaStream::CloseAudioInput(CubebUtils::AudioDeviceID aID,
+                                   AudioDataListener* aListener)
 {
+  MOZ_ASSERT(mInputListener == aListener);
   // Destroy() may have run already and cleared this
   if (GraphImpl() && mInputListener) {
-    GraphImpl()->CloseAudioInput(mInputListener);
+    GraphImpl()->CloseAudioInput(aID, aListener);
   }
   mInputListener = nullptr;
 }
 
 void
 SourceMediaStream::DestroyImpl()
 {
-  CloseAudioInput();
+  CloseAudioInput(nullptr, mInputListener);
 
   GraphImpl()->AssertOnGraphThreadOrNotRunning();
   for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
     // Disconnect before we come under mMutex's lock since it can call back
     // through RemoveDirectTrackListenerImpl() and deadlock.
     mConsumers[i]->Disconnect();
   }
 
@@ -3302,55 +3351,16 @@ SourceMediaStream::HasPendingAudioTrack(
       audioTrackPresent = true;
       break;
     }
   }
 
   return audioTrackPresent;
 }
 
-bool
-SourceMediaStream::OpenNewAudioCallbackDriver(AudioDataListener * aListener)
-{
-  // Can't AppendMessage except on Mainthread. This is an ungly trick
-  // to bounce the message in mainthread and then in MSG thread.
-  if (!NS_IsMainThread()) {
-    RefPtr<nsIRunnable> runnable =
-      WrapRunnable(this,
-                   &SourceMediaStream::OpenNewAudioCallbackDriver,
-                   aListener);
-    GraphImpl()->mAbstractMainThread->Dispatch(runnable.forget());
-    return true;
-  }
-
-  AudioCallbackDriver* nextDriver = new AudioCallbackDriver(GraphImpl());
-  nextDriver->SetInputListener(aListener);
-
-  class Message : public ControlMessage {
-  public:
-    Message(MediaStream* aStream, AudioCallbackDriver* aNextDriver)
-    : ControlMessage(aStream)
-    , mNextDriver(aNextDriver)
-    {MOZ_ASSERT(mNextDriver);}
-    void Run() override
-    {
-       MediaStreamGraphImpl* graphImpl = mNextDriver->GraphImpl();
-       MonitorAutoLock mon(graphImpl->GetMonitor());
-       MOZ_ASSERT(graphImpl->LifecycleStateRef() ==
-                  MediaStreamGraphImpl::LifecycleState::LIFECYCLE_RUNNING);
-       graphImpl->CurrentDriver()->SwitchAtNextIteration(mNextDriver);
-    }
-    AudioCallbackDriver* mNextDriver;
-  };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, nextDriver));
-
-  return true;
-}
-
-
 void
 MediaInputPort::Init()
 {
   LOG(LogLevel::Debug,
       ("Adding MediaInputPort %p (from %p to %p) to the graph",
        this,
        mSource,
        mDest));
@@ -3596,20 +3606,18 @@ ProcessedMediaStream::DestroyImpl()
   // SetStreamOrderDirty(), for other reasons.
 }
 
 MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested,
                                            TrackRate aSampleRate,
                                            AbstractThread* aMainThread)
   : MediaStreamGraph(aSampleRate)
   , mPortCount(0)
-  , mInputWanted(false)
-  , mInputDeviceID(-1)
-  , mOutputWanted(true)
-  , mOutputDeviceID(-1)
+  , mInputDeviceID(nullptr)
+  , mOutputDeviceID(nullptr)
   , mNeedAnotherIteration(false)
   , mGraphDriverAsleep(false)
   , mMonitor("MediaStreamGraphImpl")
   , mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED)
   , mEndTime(GRAPH_TIME_MAX)
   , mForceShutDown(false)
   , mPostedRunInStableStateEvent(false)
   , mDetectedNotRunning(false)
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -112,16 +112,21 @@ public:
    * Input data from a microphone (or other audio source.  This is not
    * guaranteed to be in any particular size chunks.
    */
   virtual void NotifyInputData(MediaStreamGraph* aGraph,
                                const AudioDataValue* aBuffer, size_t aFrames,
                                TrackRate aRate, uint32_t aChannels) = 0;
 
   /**
+   * Number of audio input channels.
+   */
+  virtual uint32_t InputChannelCount() = 0;
+
+  /**
    * Called when the underlying audio device has changed.
    */
   virtual void DeviceChanged() = 0;
 };
 
 class AudioDataListener : public AudioDataListenerInterface {
 protected:
   // Protected destructor, to discourage deletion outside of Release():
@@ -698,20 +703,21 @@ public:
    * it is still possible for a NotifyPull to occur.
    */
   void SetPullEnabled(bool aEnabled);
 
   // Users of audio inputs go through the stream so it can track when the
   // last stream referencing an input goes away, so it can close the cubeb
   // input.  Also note: callable on any thread (though it bounces through
   // MainThread to set the command if needed).
-  nsresult OpenAudioInput(int aID,
+  nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID,
                           AudioDataListener *aListener);
   // Note: also implied when Destroy() happens
-  void CloseAudioInput();
+  void CloseAudioInput(CubebUtils::AudioDeviceID aID,
+                       AudioDataListener* aListener);
 
   // MediaStreamGraph thread only
   void DestroyImpl() override;
 
   // Call these on any thread.
   /**
    * Call all MediaStreamListeners to request new data via the NotifyPull API
    * (if enabled).
@@ -828,18 +834,16 @@ public:
   bool HasPendingAudioTrack();
 
   TimeStamp GetStreamTracksStrartTimeStamp()
   {
     MutexAutoLock lock(mMutex);
     return mStreamTracksStartTimeStamp;
   }
 
-  bool OpenNewAudioCallbackDriver(AudioDataListener *aListener);
-
   // XXX need a Reset API
 
   friend class MediaStreamGraphImpl;
 
 protected:
   enum TrackCommands : uint32_t;
 
   virtual ~SourceMediaStream();
@@ -1314,23 +1318,20 @@ public:
 
   // Return the correct main thread for this graph. This always returns
   // something that is valid. Thread safe.
   AbstractThread* AbstractMainThread();
 
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
 
-  virtual nsresult OpenAudioInput(int aID,
-                                  AudioDataListener *aListener)
-  {
-    return NS_ERROR_FAILURE;
-  }
-  virtual void CloseAudioInput(AudioDataListener *aListener) {}
-
+  virtual nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID,
+                                  AudioDataListener *aListener) = 0;
+  virtual void CloseAudioInput(CubebUtils::AudioDeviceID aID,
+                               AudioDataListener *aListener) = 0;
   // Control API.
   /**
    * Create a stream that a media decoder (or some other source of
    * media data, such as a camera) can write to.
    */
   SourceMediaStream* CreateSourceStream();
   /**
    * Create a stream that will form the union of the tracks of its input
@@ -1398,23 +1399,16 @@ public:
   TrackRate GraphRate() const { return mSampleRate; }
 
   void RegisterCaptureStreamForWindow(uint64_t aWindowId,
                                       ProcessedMediaStream* aCaptureStream);
   void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
   already_AddRefed<MediaInputPort> ConnectToCaptureStream(
     uint64_t aWindowId, MediaStream* aMediaStream);
 
-  /**
-   * Data going to the speakers from the GraphDriver's DataCallback
-   * to notify any listeners (for echo cancellation).
-   */
-  void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
-                        TrackRate aRate, uint32_t aChannels);
-
   void AssertOnGraphThreadOrNotRunning() const
   {
     MOZ_ASSERT(OnGraphThreadOrNotRunning());
   }
 
 protected:
   explicit MediaStreamGraph(TrackRate aSampleRate)
     : mSampleRate(aSampleRate)
@@ -1435,18 +1429,13 @@ protected:
   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
 
   /**
    * Sample rate at which this graph runs. For real time graphs, this is
    * the rate of the audio mixer. For offline graphs, this is the rate specified
    * at construction.
    */
   TrackRate mSampleRate;
-
-  /**
-   * CloseAudioInput is async, so hold a reference here.
-   */
-  nsTArray<RefPtr<AudioDataListener>> mAudioInputs;
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
--- a/dom/media/MediaStreamGraphImpl.h
+++ b/dom/media/MediaStreamGraphImpl.h
@@ -12,17 +12,17 @@
 #include "GraphDriver.h"
 #include "Latency.h"
 #include "mozilla/Atomics.h"
 #include "mozilla/Monitor.h"
 #include "mozilla/Services.h"
 #include "mozilla/TimeStamp.h"
 #include "mozilla/UniquePtr.h"
 #include "mozilla/WeakPtr.h"
-#include "nsDataHashtable.h"
+#include "nsClassHashtable.h"
 #include "nsIMemoryReporter.h"
 #include "nsINamed.h"
 #include "nsIRunnable.h"
 #include "nsIThread.h"
 #include "nsITimer.h"
 #include "AsyncLogger.h"
 
 namespace mozilla {
@@ -373,27 +373,48 @@ public:
    * If aStream doesn't need an audio stream but has one, destroy it.
    */
   void CreateOrDestroyAudioStreams(MediaStream* aStream);
   /**
    * Queue audio (mix of stream audio and silence for blocked intervals)
    * to the audio output stream. Returns the number of frames played.
    */
   StreamTime PlayAudio(MediaStream* aStream);
-  /**
-   * No more data will be forthcoming for aStream. The stream will end
-   * at the current buffer end point. The StreamTracks's tracks must be
-   * explicitly set to finished by the caller.
-   */
-  void OpenAudioInputImpl(int aID,
+  /* Runs off a message on the graph thread when something requests audio from
+   * an input audio device of ID aID, and delivers the input audio frames to
+   * aListener. */
+  void OpenAudioInputImpl(CubebUtils::AudioDeviceID aID,
                           AudioDataListener *aListener);
-  virtual nsresult OpenAudioInput(int aID,
+  /* Called on the main thread when something requests audio from an input
+   * audio device aID. */
+  virtual nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID,
                                   AudioDataListener *aListener) override;
-  void CloseAudioInputImpl(AudioDataListener *aListener);
-  virtual void CloseAudioInput(AudioDataListener *aListener) override;
+  /* Runs off a message on the graph when a input audio from aID is not needed
+   * anymore, for a particular stream. It can be that other streams still need
+   * audio from this audio input device. */
+  void CloseAudioInputImpl(CubebUtils::AudioDeviceID aID,
+                           AudioDataListener *aListener);
+  /* Called on the main thread when input audio from aID is not needed
+   * anymore, for a particular stream. It can be that other streams still need
+   * audio from this audio input device. */
+  virtual void CloseAudioInput(CubebUtils::AudioDeviceID aID,
+                               AudioDataListener *aListener) override;
+  /* Called on the graph thread when the input device settings should be
+   * reevaluated, for example, if the channel count of the input stream should
+   * be changed . */
+  void ReevaluateInputDevice();
+  /* Called on the graph thread when there is new output data for listeners.
+   * This is the mixed audio output of this MediaStreamGraph. */
+  void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
+                        TrackRate aRate, uint32_t aChannels);
+  /* Called on the graph thread when there is new input data for listeners. This
+   * is the raw audio input for this MediaStreamGraph. */
+  void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
+                       TrackRate aRate, uint32_t aChannels);
+  void DeviceChanged();
 
   /**
    * Compute how much stream data we would like to buffer for aStream.
    */
   StreamTime GetDesiredBufferEnd(MediaStream* aStream);
   /**
    * Returns true when there are no active streams.
    */
@@ -422,21 +443,50 @@ public:
    * Mark the media stream order as dirty.
    */
   void SetStreamOrderDirty()
   {
     MOZ_ASSERT(OnGraphThreadOrNotRunning());
     mStreamOrderDirty = true;
   }
 
-  uint32_t AudioChannelCount() const
+  uint32_t AudioOutputChannelCount() const
   {
     return mOutputChannels;
   }
 
+  /** The audio input channel count for a MediaStreamGraph is the max of all the
+   * channel count requested by the listeners. The max channel count is
+   * delievered to the listener themselve, and they take care of downmixing. */
+  uint32_t AudioInputChannelCount()
+  {
+    MOZ_ASSERT(OnGraphThreadOrNotRunning());
+
+    if (!mInputDeviceID) {
+      MOZ_ASSERT(mInputDeviceUsers.Count() == 0);
+      return 0;
+    }
+    uint32_t maxInputChannels = 0;
+    // When/if we decide to support multiple input device per graph, this needs
+    // loop over them.
+    nsTArray<RefPtr<AudioDataListener>>* listeners =
+      mInputDeviceUsers.GetValue(mInputDeviceID);
+    MOZ_ASSERT(listeners);
+    for (auto& listener : *listeners) {
+      maxInputChannels =
+        std::max(maxInputChannels, listener->InputChannelCount());
+    }
+    return maxInputChannels;
+  }
+
+  CubebUtils::AudioDeviceID InputDeviceID()
+  {
+    return mInputDeviceID;
+  }
+
   double MediaTimeToSeconds(GraphTime aTime) const
   {
     NS_ASSERTION(aTime > -STREAM_TIME_MAX && aTime <= STREAM_TIME_MAX,
                  "Bad time");
     return static_cast<double>(aTime)/GraphRate();
   }
 
   GraphTime SecondsToMediaTime(double aS) const
@@ -620,26 +670,31 @@ public:
    */
   TimeStamp mLastMainThreadUpdate;
   /**
    * Number of active MediaInputPorts
    */
   int32_t mPortCount;
 
   /**
-   * Devices to use for cubeb input & output, or NULL for no input (void*),
-   * and boolean to control if we want input/output
+   * Devices to use for cubeb input & output, or nullptr for default device.
+   * A MediaStreamGraph always has an output (even if silent).
+   * If `mInputDeviceUsers.Count() != 0`, this MediaStreamGraph wants audio
+   * input.
+   *
+   * In any case, the number of channels to use can be queried (on the graph
+   * thread) by AudioInputChannelCount() and AudioOutputChannelCount().
    */
-  bool mInputWanted;
-  int mInputDeviceID;
-  bool mOutputWanted;
-  int mOutputDeviceID;
-  // Maps AudioDataListeners to a usecount of streams using the listener
-  // so we can know when it's no longer in use.
-  nsDataHashtable<nsPtrHashKey<AudioDataListener>, uint32_t> mInputDeviceUsers;
+  CubebUtils::AudioDeviceID mInputDeviceID;
+  CubebUtils::AudioDeviceID mOutputDeviceID;
+  // Maps AudioDeviceID to an array of their users (that are listeners). This is
+  // used to deliver audio input frames and to notify the listeners that the
+  // audio device that delivers the audio frames has changed.
+  nsDataHashtable<nsVoidPtrHashKey,
+                  nsTArray<RefPtr<AudioDataListener>>> mInputDeviceUsers;
 
   // True if the graph needs another iteration after the current iteration.
   Atomic<bool> mNeedAnotherIteration;
   // GraphDriver may need a WakeUp() if something changes
   Atomic<bool> mGraphDriverAsleep;
 
   // mMonitor guards the data below.
   // MediaStreamGraph normally does its work without holding mMonitor, so it is