Bug 1363667 - P6 - RTP Source PeerConnection JS impl draft
authorNico Grunbaum
Tue, 14 Nov 2017 10:32:07 -0800
changeset 706491 9d09adc6236f3a4bf94e7e8d9191c164908e1deb
parent 706490 2df6552617a2b4bc94516c758ea7e8829db837a1
child 706492 1db4346841a00590c4f241d90f8f1c4ea0dcffa0
push id91808
push userna-g@nostrum.com
push dateSat, 02 Dec 2017 00:48:28 +0000
bugs1363667
milestone59.0a1
Bug 1363667 - P6 - RTP Source PeerConnection JS impl MozReview-Commit-ID: CkFY5fABkr
dom/media/PeerConnection.js
media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp
media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.h
--- a/dom/media/PeerConnection.js
+++ b/dom/media/PeerConnection.js
@@ -341,16 +341,33 @@ setupPrototype(RTCStatsReport, {
         "inbound-rtp": "inboundrtp",
         "outbound-rtp": "outboundrtp",
         "candidate-pair": "candidatepair",
         "local-candidate": "localcandidate",
         "remote-candidate": "remotecandidate"
   }
 });
 
+// Cache for RTPSourceEntries
+// Note: each cache is only valid for one JS event loop execution
+class RTCRtpSourceCache {
+  constructor() {
+    // The time in RTP source time (ms)
+    this.tsNowInRtpSourceTime = null;
+    // The time in JS
+    this.jsTimestamp = null;
+    // Time difference between JS time and RTP source time
+    this.timestampOffset = null;
+    // RTPSourceEntries cached by track id
+    this.rtpSourcesByTrackId = new Map();
+    // Has a cache wipe already been scheduled
+    this.scheduledClear = null;
+  }
+}
+
 class RTCPeerConnection {
   constructor() {
     this._receiveStreams = new Map();
     this._transceivers = [];
 
     this._pc = null;
     this._closed = false;
 
@@ -361,20 +378,23 @@ class RTCPeerConnection {
     // is set to true or false based on the presence of the "trickle" ice-option
     this._canTrickle = null;
 
     // States
     this._iceGatheringState = this._iceConnectionState = "new";
 
     this._hasStunServer = this._hasTurnServer = false;
     this._iceGatheredRelayCandidates = false;
-
-  // TODO: Remove legacy API eventually
-  // see Bug 1328194
-  this._onGetStatsIsLegacy = false;
+    // Stored webrtc timing information
+    this._storedRtpSourceReferenceTime = null;
+    // TODO: Remove legacy API eventually
+    // see Bug 1328194
+    this._onGetStatsIsLegacy = false;
+    // Stores cached RTP sources state
+    this._rtpSourceCache = new RTCRtpSourceCache();
   }
 
   init(win) {
     this._win = win;
   }
 
   __init(rtcConfig) {
     this._winID = this._win.QueryInterface(Ci.nsIInterfaceRequestor)
@@ -1220,16 +1240,55 @@ class RTCPeerConnection {
     let init = {direction: "recvonly"};
     let transceiver = this._win.RTCRtpTransceiver._create(
         this._win,
         new RTCRtpTransceiver(this, transceiverImpl, init, kind, null));
     transceiver.sync();
     this._transceivers.push(transceiver);
   }
 
+  /* Returns a dictionary with three keys:
+   * sources: a list of contributing and synchronization sources
+   * sourceClockOffset: an offset to apply to the source timestamp to get a
+   * very close approximation of the sample time with respect to the local
+   * clock.
+   * jsTimestamp: the current JS time
+   * Note: because the two clocks can drift with respect to each other, once
+   *  a timestamp offset has been calculated it should not be recalculated
+   *  until the timestamp changes, this way it will not appear as if a new
+   *  audio level sample has arrived.
+   */
+  _getRtpSources(receiver) {
+    let cache = this._rtpSourceCache;
+    // Schedule cache invalidation
+    if (!cache.scheduledClear) {
+      cache.scheduledClear = true;
+      Promise.resolve().then(() => {
+        this._rtpSourceCache = new RTCRtpSourceCache();
+      });
+    }
+    // Fetch the RTP source local time, store it for reuse, calculate
+    // the local offset, likewise store it for reuse.
+    if (cache.tsNowInRtpSourceTime !== undefined) {
+      cache.tsNowInRtpSourceTime = this._impl.getNowInRtpSourceReferenceTime();
+      cache.jsTimestamp = new Date().getTime();
+      cache.timestampOffset = cache.jsTimestamp - cache.tsNowInRtpSourceTime;
+    }
+    let id = receiver.track.id;
+    if (cache.rtpSourcesByTrackId[id] === undefined) {
+      cache.rtpSourcesByTrackId[id] =
+          this._impl.getRtpSources(receiver.track, cache.tsNowInRtpSourceTime);
+    }
+    return {
+      sources: cache.rtpSourcesByTrackId[id],
+      sourceClockOffset: cache.timestampOffset,
+      jsTimestamp: cache.jsTimestamp,
+    };
+  }
+
   addTransceiver(sendTrackOrKind, init) {
     let transceiver = this._addTransceiverNoEvents(sendTrackOrKind, init);
     this.updateNegotiationNeeded();
     return transceiver;
   }
 
   _syncTransceivers() {
     this._transceivers.forEach(transceiver => transceiver.sync());
@@ -1318,16 +1377,34 @@ class RTCPeerConnection {
   getSenders() {
     return this.getTransceivers().map(transceiver => transceiver.sender);
   }
 
   getReceivers() {
     return this.getTransceivers().map(transceiver => transceiver.receiver);
   }
 
+  // test-only: get the current time using the webrtc clock
+  mozGetNowInRtpSourceReferenceTime() {
+    return this._impl.getNowInRtpSourceReferenceTime();
+  }
+
+  // test-only: insert a contributing source entry for a track
+  mozInsertAudioLevelForContributingSource(receiver,
+                                           source,
+                                           timestamp,
+                                           hasLevel,
+                                           level) {
+    this._impl.insertAudioLevelForContributingSource(receiver.track,
+                                                     source,
+                                                     timestamp,
+                                                     hasLevel,
+                                                     level);
+  }
+
   mozAddRIDExtension(receiver, extensionId) {
     this._impl.addRIDExtension(receiver.track, extensionId);
   }
 
   mozAddRIDFilter(receiver, rid) {
     this._impl.addRIDFilter(receiver.track, rid);
   }
 
@@ -1983,26 +2060,104 @@ setupPrototype(RTCRtpSender, {
 
 class RTCRtpReceiver {
   constructor(pc, transceiverImpl) {
     // We do not set the track here; that is done when _transceiverImpl is set
     Object.assign(this,
         {
           _pc: pc,
           _transceiverImpl: transceiverImpl,
-          track: transceiverImpl.getReceiveTrack()
+          track: transceiverImpl.getReceiveTrack(),
+          // Sync and contributing sources must be kept cached so that timestamps
+          // remain stable, as the timestamp offset can vary
+          // note key = entry.source + entry.sourceType
+          _rtpSources: new Map(),
+          _rtpSourcesJsTimestamp: null,
         });
   }
 
   // TODO(bug 1401983): Create a getStats binding on TransceiverImpl, and use
   // that here.
   getStats() {
     return this._pc._async(
       async () => this._pc.getStats(this.track));
   }
+
+  _getRtpSource(source, type) {
+    this._fetchRtpSources();
+    return this._rtpSources.get(type + source).entry;
+  }
+
+  /* Fetch all of the RTP Contributing and Sync sources for the receiver
+   * and store them so they are available when asked for.
+   */
+  _fetchRtpSources() {
+    if (this._rtpSourcesJsTimestamp !== null) {
+      return;
+    }
+    // Queue microtask to mark the cache as stale after this task completes
+    Promise.resolve().then(() => this._rtpSourcesJsTimestamp = null);
+    let {sources, sourceClockOffset, jsTimestamp} =
+        this._pc._getRtpSources(this);
+    this._rtpSourcesJsTimestamp = jsTimestamp;
+    for (let entry of sources) {
+      // Set the clock offset for calculating the 10-second window
+      entry.sourceClockOffset = sourceClockOffset;
+      // Store the new entries or update existing entries
+      let key =  entry.source + entry.sourceType;
+      let cached = this._rtpSources.get(key);
+      if (cached === undefined) {
+        this._rtpSources.set(key, entry);
+      } else if (cached.timestamp != entry.timestamp) {
+        // Only update if the timestamp has changed
+        // This also prevents the sourceClockOffset from changing unecessarily
+        // which could cause a value to flutter at the edge of the 10 second
+        // window.
+        this._rtpSources.set(key, entry);
+      }
+    }
+    // Clear old entries
+    let cutoffTime = this._rtpSourcesJsTimestamp - 10 * 1000;
+    let removeKeys = [];
+    for (let entry of this._rtpSources.values()) {
+      if ((entry.timestamp + entry.sourceClockOffset) < cutoffTime) {
+        removeKeys.push(entry.source + entry.sourceType);
+      }
+    }
+    for (let delKey of removeKeys) {
+      this._rtpSources.delete(delKey);
+    }
+  }
+
+
+
+  _getRtpSourcesByType(type) {
+    this._fetchRtpSources();
+    // Only return the values from within the last 10 seconds as per the spec
+    let cutoffTime = this._rtpSourcesJsTimestamp - 10 * 1000;
+    let sources = [...this._rtpSources.values()].filter(
+      (entry) => {
+        return entry.sourceType == type &&
+            (entry.timestamp + entry.sourceClockOffset) >= cutoffTime;
+      }).map(e => ({
+        source: e.source,
+        timestamp: e.timestamp + e.sourceClockOffset,
+        audioLevel: e.audioLevel,
+      }));
+      return sources;
+  }
+
+  getContributingSources() {
+    return this._getRtpSourcesByType("contributing");
+  }
+
+  getSynchronizationSources() {
+    return this._getRtpSourcesByType("synchronization");
+  }
+
 }
 setupPrototype(RTCRtpReceiver, {
   classID: PC_RECEIVER_CID,
   contractID: PC_RECEIVER_CONTRACT,
   QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports])
 });
 
 class RTCRtpTransceiver {
--- a/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp
+++ b/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp
@@ -2218,16 +2218,65 @@ PeerConnectionImpl::DumpPacket_m(size_t 
   if (!arrayBuffer.Init(jsobj)) {
     return;
   }
 
   JSErrorResult jrv;
   pco->OnPacket(level, type, sending, arrayBuffer, jrv);
 }
 
+NS_IMETHODIMP
+PeerConnectionImpl::GetRtpSources(
+    MediaStreamTrack& aRecvTrack,
+    DOMHighResTimeStamp aRtpSourceTimeNow,
+    nsTArray<dom::RTCRtpSourceEntry>& outRtpSources)
+{
+  PC_AUTO_ENTER_API_CALL(true);
+  outRtpSources.Clear();
+  std::vector<RefPtr<TransceiverImpl>>& transceivers =
+    mMedia->GetTransceivers();
+  for (RefPtr<TransceiverImpl>& transceiver : transceivers) {
+    if (transceiver->HasReceiveTrack(&aRecvTrack)) {
+      transceiver->GetRtpSources(aRtpSourceTimeNow, outRtpSources);
+      break;
+    }
+  }
+  return NS_OK;
+}
+
+DOMHighResTimeStamp
+PeerConnectionImpl::GetNowInRtpSourceReferenceTime()
+{
+  return RtpSourceObserver::NowInReportClockTime();
+}
+
+// test-only: adds fake CSRCs and audio data
+nsresult
+PeerConnectionImpl::InsertAudioLevelForContributingSource(
+    dom::MediaStreamTrack& aRecvTrack,
+    unsigned long aSource,
+    DOMHighResTimeStamp aTimestamp,
+    bool aHasLevel,
+    uint8_t aLevel)
+{
+  PC_AUTO_ENTER_API_CALL(true);
+  std::vector<RefPtr<TransceiverImpl>>& transceivers =
+    mMedia->GetTransceivers();
+  for (RefPtr<TransceiverImpl>& transceiver : transceivers) {
+    if (transceiver->HasReceiveTrack(&aRecvTrack)) {
+      transceiver->InsertAudioLevelForContributingSource(aSource,
+                                                         aTimestamp,
+                                                         aHasLevel,
+                                                         aLevel);
+      break;
+    }
+  }
+  return NS_OK;
+}
+
 nsresult
 PeerConnectionImpl::AddRIDExtension(MediaStreamTrack& aRecvTrack,
                                     unsigned short aExtensionId)
 {
   return mMedia->AddRIDExtension(aRecvTrack, aExtensionId);
 }
 
 nsresult
@@ -2414,16 +2463,17 @@ PeerConnectionImpl::InsertDTMF(Transceiv
   if (!state->mTones.IsEmpty()) {
     state->mSendTimer->InitWithNamedFuncCallback(DTMFSendTimerCallback_m, state, 0,
                                                  nsITimer::TYPE_ONE_SHOT,
                                                  "DTMFSendTimerCallback_m");
   }
   return NS_OK;
 }
 
+
 NS_IMETHODIMP
 PeerConnectionImpl::GetDTMFToneBuffer(mozilla::dom::RTCRtpSender& sender,
                                       nsAString& outToneBuffer) {
   PC_AUTO_ENTER_API_CALL(false);
 
   JSErrorResult jrv;
 
   // Retrieve track
--- a/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.h
+++ b/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.h
@@ -68,16 +68,17 @@ class NrIceTurnServer;
 class MediaPipeline;
 class TransceiverImpl;
 
 class DOMMediaStream;
 
 namespace dom {
 class RTCCertificate;
 struct RTCConfiguration;
+struct RTCRtpSourceEntry;
 class RTCDTMFSender;
 struct RTCIceServer;
 struct RTCOfferOptions;
 struct RTCRtpParameters;
 class RTCRtpSender;
 class MediaStreamTrack;
 
 #ifdef USE_FAKE_PCOBSERVER
@@ -422,16 +423,25 @@ public:
   }
 
   NS_IMETHODIMP_TO_ERRORRESULT(GetDTMFToneBuffer, ErrorResult &rv,
                                dom::RTCRtpSender& sender,
                                nsAString& outToneBuffer) {
     rv = GetDTMFToneBuffer(sender, outToneBuffer);
   }
 
+  NS_IMETHODIMP_TO_ERRORRESULT(GetRtpSources, ErrorResult &rv,
+      dom::MediaStreamTrack& aRecvTrack,
+      DOMHighResTimeStamp aRtpSourceNow,
+      nsTArray<dom::RTCRtpSourceEntry>& outRtpSources) {
+    rv = GetRtpSources(aRecvTrack, aRtpSourceNow, outRtpSources);
+  }
+
+  DOMHighResTimeStamp GetNowInRtpSourceReferenceTime();
+
   NS_IMETHODIMP_TO_ERRORRESULT(ReplaceTrackNoRenegotiation, ErrorResult &rv,
                                TransceiverImpl& aTransceiver,
                                mozilla::dom::MediaStreamTrack* aWithTrack)
   {
     rv = ReplaceTrackNoRenegotiation(aTransceiver, aWithTrack);
   }
 
   NS_IMETHODIMP_TO_ERRORRESULT(SetParameters, ErrorResult &rv,
@@ -451,16 +461,32 @@ public:
   nsresult
   SetParameters(dom::MediaStreamTrack& aTrack,
                 const std::vector<JsepTrack::JsConstraints>& aConstraints);
 
   nsresult
   GetParameters(dom::MediaStreamTrack& aTrack,
                 std::vector<JsepTrack::JsConstraints>* aOutConstraints);
 
+  // test-onlg: called from contributing sources mochitests.
+  NS_IMETHODIMP_TO_ERRORRESULT(InsertAudioLevelForContributingSource,
+                               ErrorResult &rv,
+                               dom::MediaStreamTrack& aRecvTrack,
+                               unsigned long aSource,
+                               DOMHighResTimeStamp aTimestamp,
+                               bool aHasLevel,
+                               uint8_t aLevel)
+ {
+   rv = InsertAudioLevelForContributingSource(aRecvTrack,
+                                              aSource,
+                                              aTimestamp,
+                                              aHasLevel,
+                                              aLevel);
+ }
+
   // test-only: called from simulcast mochitests.
   NS_IMETHODIMP_TO_ERRORRESULT(AddRIDExtension, ErrorResult &rv,
                                dom::MediaStreamTrack& aRecvTrack,
                                unsigned short aExtensionId)
   {
     rv = AddRIDExtension(aRecvTrack, aExtensionId);
   }