--- a/dom/bindings/moz.build
+++ b/dom/bindings/moz.build
@@ -87,16 +87,17 @@ LOCAL_INCLUDES += [
'/js/xpconnect/wrappers',
'/layout/generic',
'/layout/style',
'/layout/xul/tree',
'/media/mtransport',
'/media/webrtc/',
'/media/webrtc/signaling/src/common/time_profiling',
'/media/webrtc/signaling/src/peerconnection',
+ '/media/webrtc/trunk/',
]
DEFINES['GOOGLE_PROTOBUF_NO_RTTI'] = True
DEFINES['GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER'] = True
UNIFIED_SOURCES += [
'BindingUtils.cpp',
'CallbackInterface.cpp',--- a/dom/media/bridge/moz.build
+++ b/dom/media/bridge/moz.build
@@ -18,14 +18,15 @@ LOCAL_INCLUDES += [
'/ipc/chromium/src',
'/media/mtransport',
'/media/mtransport',
'/media/webrtc/',
'/media/webrtc/signaling/src/common/time_profiling',
'/media/webrtc/signaling/src/media-conduit',
'/media/webrtc/signaling/src/mediapipeline',
'/media/webrtc/signaling/src/peerconnection',
+ '/media/webrtc/trunk/',
]
FINAL_LIBRARY = 'xul'
if CONFIG['GNU_CXX']:
CXXFLAGS += ['-Wno-error=shadow']--- a/media/webrtc/signaling/src/media-conduit/AudioConduit.cpp
+++ b/media/webrtc/signaling/src/media-conduit/AudioConduit.cpp
@@ -280,16 +280,58 @@ bool WebrtcAudioConduit::InsertDTMFTone(
int result = 0;
if (outOfBand){
result = mChannelProxy->SendTelephoneEventOutband(eventCode, lengthMs);
}
return result != -1;
}
+void
+WebrtcAudioConduit::OnRtpPacket(const webrtc::WebRtcRTPHeader* aHeader,
+ const int64_t aTimestamp,
+ const uint32_t aJitter) {
+ mRtpSourceObserver.OnRtpPacket(aHeader, aTimestamp, aJitter);
+}
+
+void
+WebrtcAudioConduit::GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources)
+{
+ return mRtpSourceObserver.GetRtpSources(aTimeNow, outSources);
+}
+
+// test-only: inserts a CSRC entry in a RtpSourceObserver's history for
+// getContributingSources mochitests
+void InsertAudioLevelForContributingSource(RtpSourceObserver& observer,
+ uint32_t aCsrcSource,
+ int64_t aTimestamp,
+ bool aHasAudioLevel,
+ uint8_t aAudioLevel)
+{
+ using EntryType = dom::RTCRtpSourceEntryType;
+ auto key = RtpSourceObserver::GetKey(aCsrcSource, EntryType::Contributing);
+ auto& hist = observer.mRtpSources[key];
+ hist.Insert(aTimestamp, aTimestamp, aHasAudioLevel, aAudioLevel);
+}
+
+
+void
+WebrtcAudioConduit::InsertAudioLevelForContributingSource(uint32_t aCsrcSource,
+ int64_t aTimestamp,
+ bool aHasAudioLevel,
+ uint8_t aAudioLevel)
+{
+ mozilla::InsertAudioLevelForContributingSource(mRtpSourceObserver,
+ aCsrcSource,
+ aTimestamp,
+ aHasAudioLevel,
+ aAudioLevel);
+}
+
/*
* WebRTCAudioConduit Implementation
*/
MediaConduitErrorCode WebrtcAudioConduit::Init()
{
CSFLogDebug(LOGTAG, "%s this=%p", __FUNCTION__, this);
#ifdef MOZ_WIDGET_ANDROID
@@ -366,16 +408,17 @@ MediaConduitErrorCode WebrtcAudioConduit
{
CSFLogError(LOGTAG, "%s VoiceEngine Channel creation failed",__FUNCTION__);
return kMediaConduitChannelError;
}
// Needed to access TelephoneEvent APIs in 57 if we're not using Call/audio_send_stream/etc
webrtc::VoiceEngineImpl* s = static_cast<webrtc::VoiceEngineImpl*>(mVoiceEngine);
mChannelProxy = s->GetChannelProxy(mChannel);
MOZ_ASSERT(mChannelProxy);
+ mChannelProxy->SetRtpPacketObserver(this);
CSFLogDebug(LOGTAG, "%s Channel Created %d ",__FUNCTION__, mChannel);
if(mPtrVoENetwork->RegisterExternalTransport(mChannel, *this) == -1)
{
CSFLogError(LOGTAG, "%s VoiceEngine, External Transport Failed",__FUNCTION__);
return kMediaConduitTransportRegistrationFail;
}--- a/media/webrtc/signaling/src/media-conduit/AudioConduit.h
+++ b/media/webrtc/signaling/src/media-conduit/AudioConduit.h
@@ -8,16 +8,17 @@
#include "mozilla/Attributes.h"
#include "mozilla/ReentrantMonitor.h"
#include "mozilla/TimeStamp.h"
#include "nsTArray.h"
#include "MediaConduitInterface.h"
#include "MediaEngineWrapper.h"
+#include "RtpSourceObserver.h"
// Audio Engine Includes
#include "webrtc/common_types.h"
#include "webrtc/voice_engine/include/voe_base.h"
#include "webrtc/voice_engine/include/voe_volume_control.h"
#include "webrtc/voice_engine/include/voe_codec.h"
#include "webrtc/voice_engine/include/voe_file.h"
#include "webrtc/voice_engine/include/voe_network.h"
@@ -44,17 +45,18 @@ namespace mozilla {
DOMHighResTimeStamp
NTPtoDOMHighResTimeStamp(uint32_t ntpHigh, uint32_t ntpLow);
/**
* Concrete class for Audio session. Hooks up
* - media-source and target to external transport
*/
class WebrtcAudioConduit: public AudioSessionConduit
- , public webrtc::Transport
+ , public webrtc::Transport
+ , public webrtc::RtpPacketObserver
{
public:
//VoiceEngine defined constant for Payload Name Size.
static const unsigned int CODEC_PLNAME_SIZE;
/**
* APIs used by the registered external transport to this Conduit to
* feed in received RTP Frames to the VoiceEngine for decoding
@@ -246,16 +248,28 @@ public:
unsigned int* packetsSent,
uint64_t* bytesSent) override;
bool SetDtmfPayloadType(unsigned char type, int freq) override;
bool InsertDTMFTone(int channel, int eventCode, bool outOfBand,
int lengthMs, int attenuationDb) override;
+ void GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) override;
+
+ void OnRtpPacket(const webrtc::WebRtcRTPHeader* aRtpHeader,
+ const int64_t aTimestamp,
+ const uint32_t aJitter) override;
+
+ // test-only: inserts fake CSRCs and audio level data
+ void InsertAudioLevelForContributingSource(uint32_t aSource,
+ int64_t aTimestamp,
+ bool aHasLevel,
+ uint8_t aLevel);
private:
WebrtcAudioConduit(const WebrtcAudioConduit& other) = delete;
void operator=(const WebrtcAudioConduit& other) = delete;
//Local database of currently applied receive codecs
typedef std::vector<AudioCodecConfig* > RecvCodecList;
//Function to convert between WebRTC and Conduit codec structures
@@ -317,13 +331,15 @@ private:
// Current "capture" delay (really output plus input delay)
int32_t mCaptureDelay;
uint32_t mLastTimestamp;
uint32_t mSamples;
uint32_t mLastSyncLog;
+
+ RtpSourceObserver mRtpSourceObserver;
};
} // end namespace
#endif--- a/media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h
+++ b/media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h
@@ -7,16 +7,17 @@
#include "nsISupportsImpl.h"
#include "nsXPCOM.h"
#include "nsDOMNavigationTiming.h"
#include "mozilla/RefPtr.h"
#include "mozilla/RefCounted.h"
#include "mozilla/UniquePtr.h"
#include "double-conversion/utils.h" // for DISALLOW_COPY_AND_ASSIGN
+#include "RtpSourceObserver.h"
#include "CodecConfig.h"
#include "VideoTypes.h"
#include "MediaConduitErrors.h"
#include "ImageContainer.h"
#include "webrtc/call.h"
#include "webrtc/config.h"
@@ -547,11 +548,14 @@ public:
virtual MediaConduitErrorCode
EnableMIDExtension(bool enabled, uint8_t id) = 0;
virtual bool SetDtmfPayloadType(unsigned char type, int freq) = 0;
virtual bool InsertDTMFTone(int channel, int eventCode, bool outOfBand,
int lengthMs, int attenuationDb) = 0;
+ virtual void GetRtpSources(int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) = 0;
+
};
}
#endifnew file mode 100644
--- /dev/null
+++ b/media/webrtc/signaling/src/media-conduit/RtpSourceObserver.cpp
@@ -0,0 +1,167 @@
+#include "RtpSourceObserver.h"
+#include "nsThreadUtils.h"
+#include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/modules/include/module_common_types.h"
+
+namespace mozilla {
+
+using EntryType = dom::RTCRtpSourceEntryType;
+
+RtpSourceObserver::RtpSourceObserver() :
+ mLevelGuard("RtpSourceObserver::mLevelGuard") {}
+
+void
+RtpSourceObserver::OnRtpPacket(const webrtc::WebRtcRTPHeader* aHeader,
+ const int64_t aTimestamp,
+ const uint32_t aJitter)
+{
+ auto& header = aHeader->header;
+ MutexAutoLock lock(mLevelGuard);
+ {
+ mMaxJitterWindow = std::max(mMaxJitterWindow,
+ static_cast<int64_t>(aJitter) * 2);
+ const auto jitterAdjusted = aTimestamp + aJitter;
+ auto& hist = mRtpSources[GetKey(header.ssrc, EntryType::Synchronization)];
+ hist.Prune(aTimestamp);
+ // ssrc-audio-level handling
+ hist.Insert(aTimestamp, jitterAdjusted,
+ header.extension.hasAudioLevel,
+ header.extension.audioLevel);
+
+ // csrc-audio-level handling
+ const auto& list = header.extension.csrcAudioLevels;
+ for (uint8_t i = 0; i < header.numCSRCs; i++) {
+ const uint32_t& csrc = header.arrOfCSRCs[i];
+ auto& hist = mRtpSources[GetKey(csrc, EntryType::Contributing)];
+ hist.Prune(aTimestamp);
+ bool hasLevel = i < list.numAudioLevels;
+ uint8_t level = hasLevel ? list.arrOfAudioLevels[i] : 0;
+ hist.Insert(aTimestamp, jitterAdjusted, hasLevel, level);
+ }
+ }
+}
+
+void
+RtpSourceObserver::GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) const
+{
+ MutexAutoLock lock(mLevelGuard);
+ outSources.Clear();
+ for (const auto& it : mRtpSources) {
+ const RtpSourceEntry* entry = it.second.FindClosestNotAfter(aTimeNow);
+ if (entry) {
+ dom::RTCRtpSourceEntry domEntry;
+ domEntry.mSource.Construct(GetSourceFromKey(it.first));
+ domEntry.mSourceType.Construct(GetTypeFromKey(it.first));
+ domEntry.mTimestamp.Construct(entry->jitterAdjustedTimestamp);
+ if (entry->hasAudioLevel) {
+ domEntry.mAudioLevel.Construct(entry->audioLevel);
+ }
+ outSources.AppendElement(std::move(domEntry));
+ }
+ }
+}
+
+int64_t RtpSourceObserver::NowInReportClockTime() {
+ return webrtc::Clock::GetRealTimeClock()->TimeInMilliseconds();
+}
+
+const RtpSourceObserver::RtpSourceEntry*
+RtpSourceObserver::RtpSourceHistory::FindClosestNotAfter(int64_t aTime) const {
+ // This method scans the history for the entry whose timestamp is closest to a
+ // given timestamp but no greater. Because it is scanning forward, it keeps
+ // track of the closest entry it has found so far in case it overshoots.
+ // There is no before map.begin() which complicates things, so found tracks
+ // if something was really found.
+ auto lastFound = mDetailedHistory.cbegin();
+ bool found = false;
+ for (const auto& it : mDetailedHistory) {
+ if (it.second.jitterAdjustedTimestamp > aTime) {
+ break;
+ }
+ // lastFound can't start before begin, so the first inc must be skipped
+ if (found) {
+ lastFound++;
+ }
+ found = true;
+ }
+ if (found) {
+ return &lastFound->second;
+ }
+ if (HasEvicted() && aTime >= mLatestEviction.jitterAdjustedTimestamp) {
+ return &mLatestEviction;
+ }
+ return nullptr;
+}
+
+void
+RtpSourceObserver::RtpSourceHistory::Prune(const int64_t aTimeNow) {
+ const auto aTimeT = aTimeNow - mMaxJitterWindow;
+ const auto aTimePrehistory = aTimeNow - kHistoryWindow;
+ bool found = false;
+ // New lower bound of the map
+ auto lower = mDetailedHistory.begin();
+ for (auto& it : mDetailedHistory) {
+ if (it.second.jitterAdjustedTimestamp > aTimeT) {
+ found = true;
+ break;
+ }
+ if (found) {
+ lower++;
+ }
+ found = true;
+ }
+ if (found) {
+ if (lower->second.jitterAdjustedTimestamp > aTimePrehistory) {
+ mLatestEviction = lower->second;
+ mHasEvictedEntry = true;
+ }
+ lower++;
+ mDetailedHistory.erase(mDetailedHistory.begin(), lower);
+ }
+ if (HasEvicted() &&
+ (mLatestEviction.jitterAdjustedTimestamp + kHistoryWindow) < aTimeNow) {
+ mHasEvictedEntry = false;
+ }
+}
+
+void
+RtpSourceObserver::RtpSourceHistory::Insert(const int64_t aTimeNow,
+ const int64_t aTimestamp,
+ const bool aHasAudioLevel,
+ const uint8_t aAudioLevel)
+{
+ Insert(aTimeNow, aTimestamp).Update(aTimestamp, aHasAudioLevel, aAudioLevel);
+}
+
+RtpSourceObserver::RtpSourceEntry&
+RtpSourceObserver::RtpSourceHistory::Insert(const int64_t aTimeNow,
+ const int64_t aTimestamp)
+{
+ // Time T is the oldest time inside the jitter window (now - jitter)
+ // Time J is the newest time inside the jitter window (now + jitter)
+ // Time x is the jitter adjusted entry time
+ // Time Z is the time of the long term storage element
+ // Times A, B, C are times of entries in the jitter window buffer
+ // x-axis: time
+ // x or x T J
+ // |------Z-----|ABC| -> |------Z-----|ABC|
+ if ((aTimestamp + kHistoryWindow) < aTimeNow ||
+ aTimestamp < mLatestEviction.jitterAdjustedTimestamp) {
+ return mPrehistory; // A.K.A. /dev/null
+ }
+ mMaxJitterWindow = std::max(mMaxJitterWindow,
+ (aTimestamp - aTimeNow) * 2);
+ const int64_t aTimeT = aTimeNow - mMaxJitterWindow;
+ // x T J
+ // |------Z-----|ABC| -> |--------x---|ABC|
+ if (aTimestamp < aTimeT) {
+ mHasEvictedEntry = true;
+ return mLatestEviction;
+ }
+ // T X J
+ // |------Z-----|AB-C| -> |--------x---|ABXC|
+ return mDetailedHistory[aTimestamp];
+}
+
+}new file mode 100644
--- /dev/null
+++ b/media/webrtc/signaling/src/media-conduit/RtpSourceObserver.h
@@ -0,0 +1,175 @@
+#ifndef AUDIOLEVELOBSERVER_H
+#define AUDIOLEVELOBSERVER_H
+
+#include <vector>
+#include <map>
+
+#include "mozilla/Mutex.h"
+#include "nsISupportsImpl.h"
+#include "mozilla/dom/RTCRtpSourcesBinding.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_packet_observer.h"
+
+// Unit Test class
+namespace test {
+ class RtpSourcesTest;
+}
+
+namespace mozilla {
+
+/* Observes reception of RTP packets and tabulates data about the
+ * most recent arival times by source (csrc or ssrc) and audio level information
+ * * csrc-audio-level RTP header extension
+ * * ssrc-audio-level RTP header extension
+ */
+class RtpSourceObserver: public webrtc::RtpPacketObserver {
+public:
+
+ RtpSourceObserver();
+
+ virtual ~RtpSourceObserver() {};
+
+ void OnRtpPacket(const webrtc::WebRtcRTPHeader* aRtpHeader,
+ const int64_t aTimestamp,
+ const uint32_t aJitter) override;
+
+ /* Get the local time in MS from the same clock source that is used
+ * to generate the capture timestamps. Use for computing the age of
+ * an entry relative to another clock, e.g. the JS
+ * @return time of now in MS
+ */
+ static int64_t NowInReportClockTime();
+
+ /*
+ * Get the most recent 10 second window of CSRC and SSRC sources.
+ * @param aTimeNow the current report clock time, @see NowInReportClockTime.
+ * @param outLevels will be popluted with source entries
+ * Note: this takes jitter into account when calculating the window so
+ * the window is actually [time - jitter - 10 sec .. time - jitter]
+ */
+ void
+ GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) const;
+
+private:
+ // Note: these are pool allocated
+ struct RtpSourceEntry {
+ RtpSourceEntry() = default;
+ void Update(const int64_t aTimestamp,
+ const bool aHasAudioLevel,
+ const uint8_t aAudioLevel) {
+ jitterAdjustedTimestamp = aTimestamp;
+ // Audio level range is 0 - 127 inclusive
+ hasAudioLevel = aHasAudioLevel && !(aAudioLevel & 0x80);
+ audioLevel = aAudioLevel;
+ }
+ // Time this information was received + jitter
+ int64_t jitterAdjustedTimestamp = 0;
+ bool hasAudioLevel = false;
+ uint8_t audioLevel = 0;
+ };
+ /* Maintains a history of packets for reporting with getContributingSources
+ * and getSynchronizationSources. It is expected that entries will not always
+ * be observed in chronological order, and that the correct entry for a query
+ * not be the most recently added item. Many times the query time is expected
+ * to fall within [now - Jitter window .. now + Jitter Window]. A full history
+ * is kept within the jitter window, and only the most recent to fall out of
+ * the window is stored for the full 10 seconds. This value is only likely to
+ * be returned when the stream is stopped or paused.
+ * x-axis: time (non-linear scale)
+ * let J = now + Jitter Window
+ * let T = now - Jitter Window
+ * now - 10 seconds T now J
+ * |-----------------Z--------------------------|-AB--CDEFG-HI--J|
+ * ^Latest evicted ^Jitter buffer entries
+ * Ex Query Time ^Q0 ^Q1 ^Q2 ^Q3 ^Q4
+ * Query result:
+ * Q0: Nothing
+ * Q1: Z
+ * Q2: B
+ * Q3: E
+ * Q4: I
+ */
+ class RtpSourceHistory {
+ public:
+ RtpSourceHistory() = default;
+ // Finds the closest entry to a time, and passes that value to a closure
+ // Note: the pointer is invalidated by any operation on the history
+ // Note: the pointer is owned by the RtpSourceHistory
+ const RtpSourceEntry* FindClosestNotAfter(int64_t aTime) const;
+ // Inserts data into the history, may silently drop data if it is too old
+ void Insert(const int64_t aTimeNow,
+ const int64_t aTimestamp,
+ const bool aHasAudioLevel,
+ const uint8_t aAudioLevel);
+ // Removes aged out from the jitter window
+ void Prune(const int64_t aTimeNow);
+ // Set Source
+ void SetSource(uint32_t aSource, dom::RTCRtpSourceEntryType aType);
+ private:
+ // Finds a place to insert data and returns a reference to it
+ RtpSourceObserver::RtpSourceEntry&
+ Insert(const int64_t aTimeNow, const int64_t aTimestamp);
+ // Is the history buffer empty?
+ bool Empty() const { return !mDetailedHistory.size(); }
+ // Is there an evicted entry
+ bool HasEvicted() const { return mHasEvictedEntry; }
+
+ // Minimum amount of time (ms) to store a complete packet history
+ constexpr static int64_t kMinJitterWindow = 1000;
+ // Size of the history window (ms)
+ constexpr static int64_t kHistoryWindow = 10000;
+ // This is 2 x the maximum observed jitter or the min which ever is higher
+ int64_t mMaxJitterWindow = kMinJitterWindow;
+ // The least old entry to be kicked from the buffer.
+ RtpSourceEntry mLatestEviction;
+ // Is there an evicted entry?
+ bool mHasEvictedEntry = false;
+ std::map<int64_t, RtpSourceEntry> mDetailedHistory;
+ // Entry before history
+ RtpSourceEntry mPrehistory;
+ // Unit test
+ friend test::RtpSourcesTest;
+ };
+
+ // Do not copy or assign
+ RtpSourceObserver(const RtpSourceObserver&) = delete;
+ RtpSourceObserver& operator=(RtpSourceObserver const&) = delete;
+ // Returns a key for a source and a type
+ static uint64_t
+ GetKey(const uint32_t id, const dom::RTCRtpSourceEntryType aType) {
+ return (aType == dom::RTCRtpSourceEntryType::Synchronization) ?
+ (static_cast<uint64_t>(id) | (static_cast<uint64_t>(0x1) << 32)) :
+ (static_cast<uint64_t>(id));
+ }
+ // Returns the source from a key
+ static uint32_t GetSourceFromKey(const uint64_t aKey) {
+ return static_cast<uint32_t>(aKey & ~(static_cast<uint64_t>(0x1) << 32));
+ }
+ // Returns the type from a key
+ static dom::RTCRtpSourceEntryType GetTypeFromKey(const uint64_t aKey) {
+ return (aKey & (static_cast<uint64_t>(0x1) << 32))
+ ? dom::RTCRtpSourceEntryType::Synchronization
+ : dom::RTCRtpSourceEntryType::Contributing;
+ }
+ // Map CSRC to RtpSourceEntry
+ std::map<uint64_t, RtpSourceHistory> mRtpSources;
+ // 2 x the largest observed
+ int64_t mMaxJitterWindow;
+ // Guards statistics
+ mutable Mutex mLevelGuard;
+
+ // Unit test
+ friend test::RtpSourcesTest;
+
+ // Testing only
+ // Inserts additional csrc audio levels for mochitests
+ friend void InsertAudioLevelForContributingSource(
+ RtpSourceObserver& observer,
+ uint32_t aCsrcSource,
+ int64_t aTimestamp,
+ bool aHasAudioLevel,
+ uint8_t aAudioLevel);
+};
+}
+#undef NG
+#endif // AUDIOLEVELOBSERVER_H
\ No newline at end of file--- a/media/webrtc/signaling/src/media-conduit/moz.build
+++ b/media/webrtc/signaling/src/media-conduit/moz.build
@@ -17,16 +17,17 @@ LOCAL_INCLUDES += [
'/media/webrtc/signaling/src/peerconnection',
'/media/webrtc/trunk',
]
UNIFIED_SOURCES += [
'AudioConduit.cpp',
'GmpVideoCodec.cpp',
'MediaDataDecoderCodec.cpp',
+ 'RtpSourceObserver.cpp',
'VideoConduit.cpp',
'WebrtcGmpVideoCodec.cpp',
'WebrtcMediaDataDecoderCodec.cpp',
]
if CONFIG['OS_TARGET'] == 'Android':
UNIFIED_SOURCES += [
'MediaCodecVideoCodec.cpp',--- a/media/webrtc/signaling/src/peerconnection/TransceiverImpl.cpp
+++ b/media/webrtc/signaling/src/peerconnection/TransceiverImpl.cpp
@@ -698,16 +698,31 @@ TransceiverImpl::UpdateAudioConduit()
auto error = conduit->ConfigureRecvMediaCodecs(configs.values);
if (error) {
MOZ_MTLOG(ML_ERROR, mPCHandle << "[" << mMid << "]: " << __FUNCTION__ <<
" ConfigureRecvMediaCodecs failed: " << error);
return NS_ERROR_FAILURE;
}
+
+ const SdpExtmapAttributeList::Extmap* audioLevelExt =
+ details.GetExt(webrtc::RtpExtension::kAudioLevelUri);
+ if (audioLevelExt) {
+ MOZ_MTLOG(ML_DEBUG, "Calling EnableAudioLevelExtension");
+ error = conduit->EnableAudioLevelExtension(true,
+ audioLevelExt->entry,
+ true);
+
+ if (error) {
+ MOZ_MTLOG(ML_ERROR, mPCHandle << "[" << mMid << "]: " << __FUNCTION__ <<
+ " EnableAudioLevelExtension failed: " << error);
+ return NS_ERROR_FAILURE;
+ }
+ }
}
if (mJsepTransceiver->mSendTrack.GetNegotiatedDetails() &&
mJsepTransceiver->mSendTrack.GetActive()) {
const auto& details(*mJsepTransceiver->mSendTrack.GetNegotiatedDetails());
PtrVector<AudioCodecConfig> configs;
nsresult rv = NegotiatedDetailsToAudioCodecConfigs(details, &configs);
@@ -735,17 +750,19 @@ TransceiverImpl::UpdateAudioConduit()
}
// Should these be genericized like they are in the video conduit case?
const SdpExtmapAttributeList::Extmap* audioLevelExt =
details.GetExt(webrtc::RtpExtension::kAudioLevelUri);
if (audioLevelExt) {
MOZ_MTLOG(ML_DEBUG, "Calling EnableAudioLevelExtension");
- error = conduit->EnableAudioLevelExtension(true, audioLevelExt->entry);
+ error = conduit->EnableAudioLevelExtension(true,
+ audioLevelExt->entry,
+ false);
if (error) {
MOZ_MTLOG(ML_ERROR, mPCHandle << "[" << mMid << "]: " << __FUNCTION__ <<
" EnableAudioLevelExtension failed: " << error);
return NS_ERROR_FAILURE;
}
}
@@ -1067,9 +1084,37 @@ TransceiverImpl::Stop()
}
bool
TransceiverImpl::IsVideo() const
{
return mJsepTransceiver->GetMediaType() == SdpMediaSection::MediaType::kVideo;
}
+void TransceiverImpl::GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) const
+{
+ if (IsVideo()) {
+ return;
+ }
+ WebrtcAudioConduit *audio_conduit =
+ static_cast<WebrtcAudioConduit*>(mConduit.get());
+ audio_conduit->GetRtpSources(aTimeNow, outSources);
+}
+
+
+void TransceiverImpl::InsertAudioLevelForContributingSource(uint32_t aSource,
+ int64_t aTimestamp,
+ bool aHasLevel,
+ uint8_t aLevel)
+{
+ if (IsVideo()) {
+ return;
+ }
+ WebrtcAudioConduit *audio_conduit =
+ static_cast<WebrtcAudioConduit*>(mConduit.get());
+ audio_conduit->InsertAudioLevelForContributingSource(aSource,
+ aTimestamp,
+ aHasLevel,
+ aLevel);
+}
+
} // namespace mozilla--- a/media/webrtc/signaling/src/peerconnection/TransceiverImpl.h
+++ b/media/webrtc/signaling/src/peerconnection/TransceiverImpl.h
@@ -28,16 +28,17 @@ class MediaPipelineReceive;
class MediaPipelineTransmit;
class MediaPipeline;
class MediaPipelineFilter;
class WebRtcCallWrapper;
class JsepTrackNegotiatedDetails;
namespace dom {
class RTCRtpTransceiver;
+struct RTCRtpSourceEntry;
}
/**
* This is what ties all the various pieces that make up a transceiver
* together. This includes:
* DOMMediaStream, MediaStreamTrack, SourceMediaStream for rendering and capture
* TransportFlow for RTP transmission/reception
* Audio/VideoConduit for feeding RTP/RTCP into webrtc.org for decoding, and
@@ -104,16 +105,25 @@ public:
RefPtr<MediaPipeline> GetReceivePipeline();
void AddRIDExtension(unsigned short aExtensionId);
void AddRIDFilter(const nsAString& aRid);
bool IsVideo() const;
+ void GetRtpSources(const int64_t aTimeNow,
+ nsTArray<dom::RTCRtpSourceEntry>& outSources) const;
+
+ // test-only: insert fake CSRCs and audio levels for testing
+ void InsertAudioLevelForContributingSource(uint32_t aSource,
+ int64_t aTimestamp,
+ bool aHasLevel,
+ uint8_t aLevel);
+
NS_DECL_THREADSAFE_ISUPPORTS
private:
virtual ~TransceiverImpl();
void InitAudio();
void InitVideo();
nsresult UpdateAudioConduit();
nsresult UpdateVideoConduit();new file mode 100644
--- /dev/null
+++ b/media/webrtc/trunk/webrtc/modules/rtp_rtcp/include/rtp_packet_observer.h
@@ -0,0 +1,19 @@
+#ifndef RTP_AUDIO_LEVEL_OBSERVER_H
+#define RTP_AUDIO_LEVEL_OBSERVER_H
+
+namespace webrtc {
+
+struct WebRtcRTPHeader;
+
+class RtpPacketObserver {
+ public:
+
+ virtual void
+ OnRtpPacket(const WebRtcRTPHeader* aRtpHeader,
+ const int64_t aTimestamp,
+ const uint32_t aJitter) = 0;
+};
+
+}
+
+#endif // RTP_AUDIO_LEVEL_OBSERVER_H--- a/media/webrtc/trunk/webrtc/voice_engine/channel.cc
+++ b/media/webrtc/trunk/webrtc/voice_engine/channel.cc
@@ -24,16 +24,17 @@
#include "webrtc/base/timeutils.h"
#include "webrtc/config.h"
#include "webrtc/logging/rtc_event_log/rtc_event_log.h"
#include "webrtc/modules/audio_coding/codecs/audio_format_conversion.h"
#include "webrtc/modules/audio_device/include/audio_device.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/pacing/packet_router.h"
+#include "webrtc/modules/rtp_rtcp/include/rtp_packet_observer.h"
#include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_receiver_strategy.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/system_wrappers/include/trace.h"
#include "webrtc/voice_engine/include/voe_external_media.h"
#include "webrtc/voice_engine/include/voe_rtp_rtcp.h"
@@ -591,16 +592,20 @@ bool Channel::GetRTCPReceiverStatistics(
if (sentPackets) {
uint64_t sentBytes = rtpCounters.MediaPayloadBytes();
*bytesReceived = sentBytes * (*packetsReceived) / sentPackets;
}
}
return true;
}
+void Channel::SetRtpPacketObserver(RtpPacketObserver* observer) {
+ rtp_source_observer_ = observer;
+}
+
int32_t Channel::SendData(FrameType frameType,
uint8_t payloadType,
uint32_t timeStamp,
const uint8_t* payloadData,
size_t payloadSize,
const RTPFragmentationHeader* fragmentation) {
WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::SendData(frameType=%u, payloadType=%u, timeStamp=%u,"
@@ -776,17 +781,28 @@ int32_t Channel::OnReceivedPayloadData(c
// Push the incoming payload (parsed and ready for decoding) into the ACM
if (audio_coding_->IncomingPacket(payloadData, payloadSize, *rtpHeader) !=
0) {
_engineStatisticsPtr->SetLastError(
VE_AUDIO_CODING_MODULE_ERROR, kTraceWarning,
"Channel::OnReceivedPayloadData() unable to push data to the ACM");
return -1;
}
-
+ // Observe incoming packets for getContributingSources and
+ // getSynchronizationSources.
+ if (rtp_source_observer_) {
+ const auto playoutFrequency = audio_coding_->PlayoutFrequency();
+ uint32_t jitter = 0;
+ if (playoutFrequency > 0) {
+ const ChannelStatistics stats = statistics_proxy_->GetStats();
+ jitter = stats.rtcp.jitter / (playoutFrequency / 1000);
+ }
+ rtp_source_observer_->OnRtpPacket(rtpHeader,
+ webrtc::Clock::GetRealTimeClock()->TimeInMilliseconds(), jitter);
+ }
int64_t round_trip_time = 0;
_rtpRtcpModule->RTT(rtp_receiver_->SSRC(), &round_trip_time, NULL, NULL,
NULL);
std::vector<uint16_t> nack_list = audio_coding_->GetNackList(round_trip_time);
if (!nack_list.empty()) {
// Can't use nack_list.data() since it's not supported by all
// compilers.--- a/media/webrtc/trunk/webrtc/voice_engine/channel.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/channel.h
@@ -45,16 +45,17 @@ namespace webrtc {
class AudioDeviceModule;
class FileWrapper;
class PacketRouter;
class ProcessThread;
class RateLimiter;
class ReceiveStatistics;
class RemoteNtpTimeEstimator;
class RtcEventLog;
+class RtpPacketObserver;
class RTPPayloadRegistry;
class RtpReceiver;
class RTPReceiverAudio;
class RtpRtcp;
class TelephoneEventHandler;
class VoEMediaProcess;
class VoERTPObserver;
class VoiceEngineObserver;
@@ -427,16 +428,18 @@ class Channel
bool GetRTCPReceiverStatistics(int64_t* timestamp,
uint32_t* jitterMs,
uint32_t* cumulativeLost,
uint32_t* packetsReceived,
uint64_t* bytesReceived,
double* packetsFractionLost,
int64_t* rtt) const;
+ virtual void SetRtpPacketObserver(RtpPacketObserver* observer);
+
protected:
void OnIncomingFractionLoss(int fraction_lost);
void OnIncomingReceiverReports(const ReportBlockList& aReportBlocks,
const int64_t aRoundTripTime,
const int64_t aReceptionTime);
private:
bool ReceivePacket(const uint8_t* packet,
@@ -565,14 +568,16 @@ class Channel
PacketRouter* packet_router_ = nullptr;
std::unique_ptr<TransportFeedbackProxy> feedback_observer_proxy_;
std::unique_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_;
std::unique_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_;
std::unique_ptr<RateLimiter> retransmission_rate_limiter_;
// TODO(ossu): Remove once GetAudioDecoderFactory() is no longer needed.
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+
+ RtpPacketObserver* rtp_source_observer_ = nullptr;
};
} // namespace voe
} // namespace webrtc
#endif // WEBRTC_VOICE_ENGINE_CHANNEL_H_--- a/media/webrtc/trunk/webrtc/voice_engine/channel_proxy.cc
+++ b/media/webrtc/trunk/webrtc/voice_engine/channel_proxy.cc
@@ -288,16 +288,20 @@ void ChannelProxy::DisassociateSendChann
RTC_DCHECK(thread_checker_.CalledOnValidThread());
channel()->set_associate_send_channel(ChannelOwner(nullptr));
}
void ChannelProxy::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
channel()->SetRtcpRttStats(rtcp_rtt_stats);
}
+void ChannelProxy::SetRtpPacketObserver(RtpPacketObserver* observer) {
+ RTC_DCHECK(thread_checker_.CalledOnValidThread());
+ channel()->SetRtpPacketObserver(observer);
+}
Channel* ChannelProxy::channel() const {
RTC_DCHECK(channel_owner_.channel());
return channel_owner_.channel();
}
} // namespace voe
} // namespace webrtc--- a/media/webrtc/trunk/webrtc/voice_engine/channel_proxy.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/channel_proxy.h
@@ -23,16 +23,17 @@
#include <vector>
namespace webrtc {
class AudioSinkInterface;
class PacketRouter;
class RtcEventLog;
class RtcpRttStats;
+class RtpPacketObserver;
class RtpPacketSender;
class Transport;
class TransportFeedbackObserver;
namespace voe {
class Channel;
@@ -106,16 +107,19 @@ class ChannelProxy {
AudioFrame* audio_frame);
virtual int NeededFrequency() const;
virtual void SetTransportOverhead(int transport_overhead_per_packet);
virtual void AssociateSendChannel(const ChannelProxy& send_channel_proxy);
virtual void DisassociateSendChannel();
virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);
+ virtual void SetRtpPacketObserver(
+ RtpPacketObserver* observer);
+
private:
Channel* channel() const;
rtc::ThreadChecker thread_checker_;
rtc::RaceChecker race_checker_;
ChannelOwner channel_owner_;
RTC_DISALLOW_COPY_AND_ASSIGN(ChannelProxy);