Bug 1221587: patch 5 - Base update of the MSG API for full-duplex r?padenot
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@@ -881,16 +881,33 @@ AudioCallbackDriver::DataCallback(AudioD
stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
} else {
NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
stillProcessing = true;
}
mBuffer.BufferFilled();
+ // Callback any observers for the AEC speaker data. Note that one
+ // (maybe) of these will be full-duplex, the others will get their input
+ // data off separate cubeb callbacks. Take care with how stuff is
+ // removed/added to this list and TSAN issues, but input and output will
+ // use separate callback methods.
+ mGraphImpl->NotifySpeakerData(aOutputBuffer, static_cast<size_t>(aFrames),
+ ChannelCount);
+
+ // Process mic data if any/needed -- after inserting far-end data for AEC!
+ if (aInputBuffer) {
+ if (mAudioInput) { // for this specific input-only or full-duplex stream
+ mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer,
+ static_cast<size_t>(aFrames),
+ ChannelCount);
+ }
+ }
+
bool switching = false;
{
MonitorAutoLock mon(mGraphImpl->GetMonitor());
switching = !!NextDriver();
}
if (switching && stillProcessing) {
// If the audio stream has not been started by the previous driver or
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@@ -191,16 +191,26 @@ public:
void EnsureNextIterationLocked();
MediaStreamGraphImpl* GraphImpl() {
return mGraphImpl;
}
virtual bool OnThread() = 0;
+ // XXX Thread-safety! Do these via commands to avoid TSAN issues
+ // and crashes!!!
+ virtual void SetInputListener(MediaStreamListener *aListener) {
+ mAudioInput = aListener;
+ }
+ // XXX do we need the param? probably no
+ virtual void RemoveInputListener(MediaStreamListener *aListener) {
+ mAudioInput = nullptr;
+ }
+
protected:
GraphTime StateComputedTime() const;
// Time of the start of this graph iteration. This must be accessed while
// having the monitor.
GraphTime mIterationStart;
// Time of the end of this graph iteration. This must be accessed while having
// the monitor.
@@ -221,16 +231,19 @@ protected:
WAITSTATE_WAITING_INDEFINITELY,
// Something has signaled RunThread() to wake up immediately,
// but it hasn't done so yet
WAITSTATE_WAKING_UP
};
// This must be access with the monitor.
WaitState mWaitState;
+ // Callback for mic data, if any
+ RefPtr<MediaStreamListener> mAudioInput;
+
// This is used on the main thread (during initialization), and the graph
// thread. No monitor needed because we know the graph thread does not run
// during the initialization.
TimeStamp mCurrentTimeStamp;
// This is non-null only when this driver has recently switched from an other
// driver, and has not cleaned it up yet (for example because the audio stream
// is currently calling the callback during initialization).
//
@@ -483,16 +496,18 @@ private:
* This is written on the previous driver's thread (if switching) or main
* thread (if this driver is the first one).
* This is read on previous driver's thread (during callbacks from
* cubeb_stream_init) and the audio thread (when switching away from this
* driver back to a SystemClockDriver).
* This is synchronized by the Graph's monitor.
* */
bool mStarted;
+ /* Listener for mic input, if any. */
+ RefPtr<MediaStreamListener> mAudioInput;
struct AutoInCallback
{
explicit AutoInCallback(AudioCallbackDriver* aDriver);
~AutoInCallback();
AudioCallbackDriver* mDriver;
};
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -25,16 +25,17 @@
#include "mozilla/dom/AudioContextBinding.h"
#include <algorithm>
#include "DOMMediaStream.h"
#include "GeckoProfiler.h"
#include "mozilla/unused.h"
#ifdef MOZ_WEBRTC
#include "AudioOutputObserver.h"
#endif
+#include "mtransport/runnable_utils.h"
#include "webaudio/blink/HRTFDatabaseLoader.h"
using namespace mozilla::layers;
using namespace mozilla::dom;
using namespace mozilla::gfx;
namespace mozilla {
@@ -917,16 +918,95 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
// If the stream has finished and the timestamps of all frames have expired
// then no more updates are required.
if (aStream->mFinished && !haveMultipleImages) {
aStream->mLastPlayedVideoFrame.SetNull();
}
}
+void
+MediaStreamGraphImpl::OpenAudioInputImpl(char *aName, MediaStreamListener *aListener)
+{
+ if (CurrentDriver()->AsAudioCallbackDriver()) {
+ CurrentDriver()->SetInputListener(aListener);
+ } else {
+ // XXX Switch to callback driver
+ }
+ mAudioInputs.AppendElement(aListener); // always monitor speaker data
+}
+
+nsresult
+MediaStreamGraphImpl::OpenAudioInput(char *aName, MediaStreamListener *aListener)
+{
+ // XXX So, so, so annoying. Can't AppendMessage except on Mainthread
+ if (!NS_IsMainThread()) {
+ NS_DispatchToMainThread(WrapRunnable(this,
+ &MediaStreamGraphImpl::OpenAudioInput,
+ aName, aListener)); // XXX Fix! string need to copied
+ return NS_OK;
+ }
+ class Message : public ControlMessage {
+ public:
+ Message(MediaStreamGraphImpl *aGraph, char *aName, MediaStreamListener *aListener) :
+ ControlMessage(nullptr), mGraph(aGraph), mName(aName), mListener(aListener) {}
+ virtual void Run()
+ {
+ mGraph->OpenAudioInputImpl(mName, mListener);
+ }
+ MediaStreamGraphImpl *mGraph;
+ char *mName; // XXX needs to copy
+ MediaStreamListener *mListener;
+ };
+ this->AppendMessage(new Message(this, aName, aListener));
+ return NS_OK;
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInputImpl(MediaStreamListener *aListener)
+{
+ CurrentDriver()->RemoveInputListener(aListener);
+ mAudioInputs.RemoveElement(aListener);
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInput(MediaStreamListener *aListener)
+{
+ // XXX So, so, so annoying. Can't AppendMessage except on Mainthread
+ if (!NS_IsMainThread()) {
+ NS_DispatchToMainThread(WrapRunnable(this,
+ &MediaStreamGraphImpl::CloseAudioInput,
+ aListener));
+ return;
+ }
+ class Message : public ControlMessage {
+ public:
+ Message(MediaStreamGraphImpl *aGraph, MediaStreamListener *aListener) :
+ ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
+ virtual void Run()
+ {
+ mGraph->CloseAudioInputImpl(mListener);
+ }
+ MediaStreamGraphImpl *mGraph;
+ MediaStreamListener *mListener;
+ };
+ this->AppendMessage(new Message(this, aListener));
+}
+
+
+// All AudioInput listeners get the same speaker data (at least for now).
+void
+MediaStreamGraph::NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+ uint32_t aChannels)
+{
+ for (auto& listener : mAudioInputs) {
+ listener->NotifySpeakerData(this, aBuffer, aFrames, aChannels);
+ }
+}
+
bool
MediaStreamGraphImpl::ShouldUpdateMainThread()
{
if (mRealtime) {
return true;
}
TimeStamp now = TimeStamp::Now();
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -176,16 +176,33 @@ public:
TrackID aInputTrackID = TRACK_INVALID) {}
/**
* Notify that all new tracks this iteration have been created.
* This is to ensure that tracks added atomically to MediaStreamGraph
* are also notified of atomically to MediaStreamListeners.
*/
virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
+
+ /* These are for cubeb audio input & output streams: */
+ /**
+ * Output data to speakers, for use as the "far-end" data for echo
+ * cancellation. This is not guaranteed to be in any particular size
+ * chunks.
+ */
+ virtual void NotifySpeakerData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ uint32_t aChannels) {}
+ /**
+ * Input data from a microphone (or other audio source. This is not
+ * guaranteed to be in any particular size chunks.
+ */
+ virtual void NotifyInputData(MediaStreamGraph* aGraph,
+ AudioDataValue* aBuffer, size_t aFrames,
+ uint32_t aChannels) {}
};
/**
* This is a base class for media graph thread listener direct callbacks
* from within AppendToTrack(). Note that your regular listener will
* still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
* you must be careful to ignore them if AddDirectListener was successful.
*/
@@ -1170,16 +1187,21 @@ public:
};
// Main thread only
static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
dom::AudioChannel aChannel);
static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
// Idempotent
static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
+ virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) {
+ return NS_ERROR_FAILURE;
+ }
+ virtual void CloseAudioInput(MediaStreamListener *aListener) {}
+
// Control API.
/**
* Create a stream that a media decoder (or some other source of
* media data, such as a camera) can write to.
*/
SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
/**
* Create a stream that will form the union of the tracks of its input
@@ -1249,16 +1271,23 @@ public:
TrackRate GraphRate() const { return mSampleRate; }
void RegisterCaptureStreamForWindow(uint64_t aWindowId,
ProcessedMediaStream* aCaptureStream);
void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
already_AddRefed<MediaInputPort> ConnectToCaptureStream(
uint64_t aWindowId, MediaStream* aMediaStream);
+ /**
+ * Data going to the speakers from the GraphDriver's DataCallback
+ * to notify any listeners (for echo cancellation).
+ */
+ void NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+ uint32_t aChannels);
+
protected:
explicit MediaStreamGraph(TrackRate aSampleRate)
: mSampleRate(aSampleRate)
{
MOZ_COUNT_CTOR(MediaStreamGraph);
}
virtual ~MediaStreamGraph()
{
@@ -1269,13 +1298,15 @@ protected:
nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
/**
* Sample rate at which this graph runs. For real time graphs, this is
* the rate of the audio mixer. For offline graphs, this is the rate specified
* at construction.
*/
TrackRate mSampleRate;
+
+ nsTArray<RefPtr<MediaStreamListener>> mAudioInputs;
};
} // namespace mozilla
#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
--- a/dom/media/MediaStreamGraphImpl.h
+++ b/dom/media/MediaStreamGraphImpl.h
@@ -345,16 +345,21 @@ public:
* Set the correct current video frame for stream aStream.
*/
void PlayVideo(MediaStream* aStream);
/**
* No more data will be forthcoming for aStream. The stream will end
* at the current buffer end point. The StreamBuffer's tracks must be
* explicitly set to finished by the caller.
*/
+ void OpenAudioInputImpl(char *aName, MediaStreamListener *aListener);
+ virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) override;
+ void CloseAudioInputImpl(MediaStreamListener *aListener);
+ virtual void CloseAudioInput(MediaStreamListener *aListener) override;
+
void FinishStream(MediaStream* aStream);
/**
* Compute how much stream data we would like to buffer for aStream.
*/
StreamTime GetDesiredBufferEnd(MediaStream* aStream);
/**
* Returns true when there are no active streams.
*/