Bug 1324548 - Implement MediaStreamTrackAudioSourceNode. r?padenot,pehrsons,baku draft
authorLéo Paquet
Thu, 19 Oct 2017 10:32:28 +0200
changeset 692832 3215c0cdb50b9e6c8cff6bf4f141f5f1766c44bf
parent 692714 4ada8f0d5cc011af4bc0f4fadbb25ea3e1e62bfc
child 738858 be2fbcef31f516ea3a8c80f507078408ba0db6c0
push id87610
push userpaul@paul.cx
push dateFri, 03 Nov 2017 16:25:41 +0000
reviewerspadenot, pehrsons, baku
bugs1324548
milestone58.0a1
Bug 1324548 - Implement MediaStreamTrackAudioSourceNode. r?padenot,pehrsons,baku MozReview-Commit-ID: AgEpuIv3G2D
dom/media/webaudio/MediaStreamTrackAudioSourceNode.cpp
dom/media/webaudio/MediaStreamTrackAudioSourceNode.h
dom/media/webaudio/moz.build
dom/media/webaudio/test/mochitest.ini
dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNode.html
dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNodeVideo.html
dom/webidl/MediaStreamTrackAudioSourceNode.webidl
dom/webidl/moz.build
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/MediaStreamTrackAudioSourceNode.cpp
@@ -0,0 +1,212 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamTrackAudioSourceNode.h"
+#include "mozilla/dom/MediaStreamTrackAudioSourceNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeExternalInputStream.h"
+#include "AudioStreamTrack.h"
+#include "nsIDocument.h"
+#include "mozilla/CORSMode.h"
+#include "nsContentUtils.h"
+#include "nsIScriptError.h"
+
+namespace mozilla {
+namespace dom {
+
+NS_IMPL_CYCLE_COLLECTION_CLASS(MediaStreamTrackAudioSourceNode)
+
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(MediaStreamTrackAudioSourceNode)
+  tmp->Destroy();
+  NS_IMPL_CYCLE_COLLECTION_UNLINK(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(AudioNode)
+
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(MediaStreamTrackAudioSourceNode)
+NS_INTERFACE_MAP_END_INHERITING(AudioNode)
+
+NS_IMPL_ADDREF_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+NS_IMPL_RELEASE_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+
+MediaStreamTrackAudioSourceNode::MediaStreamTrackAudioSourceNode(AudioContext* aContext)
+  : AudioNode(aContext,
+              2,
+              ChannelCountMode::Max,
+              ChannelInterpretation::Speakers)
+{
+}
+
+/* static */ already_AddRefed<MediaStreamTrackAudioSourceNode>
+MediaStreamTrackAudioSourceNode::Create(AudioContext& aAudioContext,
+                                   const MediaStreamTrackAudioSourceOptions& aOptions,
+                                   ErrorResult& aRv)
+{
+  if (aAudioContext.IsOffline()) {
+    aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+    return nullptr;
+  }
+
+  if (aAudioContext.CheckClosed(aRv)) {
+    return nullptr;
+  }
+
+  RefPtr<MediaStreamTrackAudioSourceNode> node =
+    new MediaStreamTrackAudioSourceNode(&aAudioContext);
+
+  node->Init(aOptions.mMediaStreamTrack, aRv);
+  if (aRv.Failed()) {
+    return nullptr;
+  }
+
+  return node.forget();
+}
+
+void
+MediaStreamTrackAudioSourceNode::Init(MediaStreamTrack* aMediaStreamTrack, ErrorResult& aRv)
+{
+  if (!aMediaStreamTrack) {
+    aRv.Throw(NS_ERROR_FAILURE);
+    return;
+  }
+
+  if (!aMediaStreamTrack->AsAudioStreamTrack()) {
+    aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
+    return;
+  }
+
+  MediaStreamGraph* graph = Context()->Graph();
+
+  AudioNodeEngine* engine = new MediaStreamTrackAudioSourceNodeEngine(this);
+  mStream = AudioNodeExternalInputStream::Create(graph, engine);
+
+  AttachToTrack(aMediaStreamTrack);
+}
+
+void
+MediaStreamTrackAudioSourceNode::Destroy()
+{
+  DetachFromTrack();
+}
+
+MediaStreamTrackAudioSourceNode::~MediaStreamTrackAudioSourceNode()
+{
+  Destroy();
+}
+
+void
+MediaStreamTrackAudioSourceNode::AttachToTrack(const RefPtr<MediaStreamTrack>& aTrack)
+{
+  MOZ_ASSERT(!mInputTrack);
+  MOZ_ASSERT(aTrack->AsAudioStreamTrack());
+
+  if (!mStream) {
+    return;
+  }
+
+  mInputTrack = aTrack;
+  ProcessedMediaStream* outputStream =
+    static_cast<ProcessedMediaStream*>(mStream.get());
+  mInputPort = mInputTrack->ForwardTrackContentsTo(outputStream);
+  PrincipalChanged(mInputTrack); // trigger enabling/disabling of the connector
+  mInputTrack->AddPrincipalChangeObserver(this);
+}
+
+void
+MediaStreamTrackAudioSourceNode::DetachFromTrack()
+{
+  if (mInputTrack) {
+    mInputTrack->RemovePrincipalChangeObserver(this);
+    mInputTrack = nullptr;
+  }
+  if (mInputPort) {
+    mInputPort->Destroy();
+    mInputPort = nullptr;
+  }
+}
+
+/**
+ * Changes the principal. Note that this will be called on the main thread, but
+ * changes will be enacted on the MediaStreamGraph thread. If the principal
+ * change results in the document principal losing access to the stream, then
+ * there needs to be other measures in place to ensure that any media that is
+ * governed by the new stream principal is not available to the MediaStreamGraph
+ * before this change completes. Otherwise, a site could get access to
+ * media that they are not authorized to receive.
+ *
+ * One solution is to block the altered content, call this method, then dispatch
+ * another change request to the MediaStreamGraph thread that allows the content
+ * under the new principal to flow. This might be unnecessary if the principal
+ * change is changing to be the document principal.
+ */
+void
+MediaStreamTrackAudioSourceNode::PrincipalChanged(MediaStreamTrack* aMediaStreamTrack)
+{
+  MOZ_ASSERT(aMediaStreamTrack == mInputTrack);
+
+  bool subsumes = false;
+  nsIDocument* doc = nullptr;
+  if (nsPIDOMWindowInner* parent = Context()->GetParentObject()) {
+    doc = parent->GetExtantDoc();
+    if (doc) {
+      nsIPrincipal* docPrincipal = doc->NodePrincipal();
+      nsIPrincipal* trackPrincipal = aMediaStreamTrack->GetPrincipal();
+      if (!trackPrincipal || NS_FAILED(docPrincipal->Subsumes(trackPrincipal, &subsumes))) {
+        subsumes = false;
+      }
+    }
+  }
+  auto stream = static_cast<AudioNodeExternalInputStream*>(mStream.get());
+  bool enabled = subsumes || aMediaStreamTrack->GetCORSMode() != CORS_NONE;
+  stream->SetInt32Parameter(MediaStreamTrackAudioSourceNodeEngine::ENABLE, enabled);
+
+  if (!enabled && doc) {
+    nsContentUtils::ReportToConsole(nsIScriptError::warningFlag,
+                                    NS_LITERAL_CSTRING("Web Audio"),
+                                    doc,
+                                    nsContentUtils::eDOM_PROPERTIES,
+                                    CrossOriginErrorString());
+  }
+}
+
+size_t
+MediaStreamTrackAudioSourceNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
+{
+  // Future:
+  // - mInputStream
+  size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf);
+  if (mInputPort) {
+    amount += mInputPort->SizeOfIncludingThis(aMallocSizeOf);
+  }
+  return amount;
+}
+
+size_t
+MediaStreamTrackAudioSourceNode::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
+{
+  return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+
+void
+MediaStreamTrackAudioSourceNode::DestroyMediaStream()
+{
+  if (mInputPort) {
+    mInputPort->Destroy();
+    mInputPort = nullptr;
+  }
+  AudioNode::DestroyMediaStream();
+}
+
+JSObject*
+MediaStreamTrackAudioSourceNode::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
+{
+  return MediaStreamTrackAudioSourceNodeBinding::Wrap(aCx, this, aGivenProto);
+}
+
+} // namespace dom
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/MediaStreamTrackAudioSourceNode.h
@@ -0,0 +1,108 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaStreamTrackAudioSourceNode_h_
+#define MediaStreamTrackAudioSourceNode_h_
+
+#include "AudioNode.h"
+#include "DOMMediaStream.h"
+#include "AudioNodeEngine.h"
+
+namespace mozilla {
+
+namespace dom {
+
+class AudioContext;
+struct MediaStreamTrackAudioSourceOptions;
+
+class MediaStreamTrackAudioSourceNodeEngine final : public AudioNodeEngine
+{
+public:
+  explicit MediaStreamTrackAudioSourceNodeEngine(AudioNode* aNode)
+    : AudioNodeEngine(aNode), mEnabled(false) {}
+
+  bool IsEnabled() const { return mEnabled; }
+  enum Parameters {
+    ENABLE
+  };
+  void SetInt32Parameter(uint32_t aIndex, int32_t aValue) override
+  {
+    switch (aIndex) {
+    case ENABLE:
+      mEnabled = !!aValue;
+      break;
+    default:
+      NS_ERROR("MediaStreamTrackAudioSourceNodeEngine bad parameter index");
+    }
+  }
+
+private:
+  bool mEnabled;
+};
+
+class MediaStreamTrackAudioSourceNode : public AudioNode,
+                                   public DOMMediaStream::TrackListener,
+                                   public PrincipalChangeObserver<MediaStreamTrack>
+{
+public:
+  static already_AddRefed<MediaStreamTrackAudioSourceNode>
+  Create(AudioContext& aContext, const MediaStreamTrackAudioSourceOptions& aOptions,
+         ErrorResult& aRv);
+
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+
+  static already_AddRefed<MediaStreamTrackAudioSourceNode>
+  Constructor(const GlobalObject& aGlobal, AudioContext& aAudioContext,
+              const MediaStreamTrackAudioSourceOptions& aOptions, ErrorResult& aRv)
+  {
+    return Create(aAudioContext, aOptions, aRv);
+  }
+
+  JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
+
+  void DestroyMediaStream() override;
+
+  uint16_t NumberOfInputs() const override { return 0; }
+
+  const char* NodeType() const override
+  {
+    return "MediaStreamTrackAudioSourceNode";
+  }
+
+  virtual const char* CrossOriginErrorString() const
+  {
+    return "MediaStreamTrackAudioSourceNodeCrossOrigin";
+  }
+
+  size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
+  size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
+
+  // Attaches to aTrack so that its audio content will be used as input.
+  void AttachToTrack(const RefPtr<MediaStreamTrack>& aTrack);
+
+  // Detaches from the currently attached track if there is one.
+  void DetachFromTrack();
+
+  // From PrincipalChangeObserver<MediaStreamTrack>.
+  void PrincipalChanged(MediaStreamTrack* aMediaStreamTrack) override;
+
+protected:
+  explicit MediaStreamTrackAudioSourceNode(AudioContext* aContext);
+  void Init(MediaStreamTrack* aMediaStreamTrack, ErrorResult& aRv);
+  void Destroy();
+  virtual ~MediaStreamTrackAudioSourceNode();
+
+private:
+  RefPtr<MediaInputPort> mInputPort;
+
+  RefPtr<MediaStreamTrack> mInputTrack;
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif
--- a/dom/media/webaudio/moz.build
+++ b/dom/media/webaudio/moz.build
@@ -58,16 +58,17 @@ EXPORTS.mozilla.dom += [
     'ConvolverNode.h',
     'DelayNode.h',
     'DynamicsCompressorNode.h',
     'GainNode.h',
     'IIRFilterNode.h',
     'MediaElementAudioSourceNode.h',
     'MediaStreamAudioDestinationNode.h',
     'MediaStreamAudioSourceNode.h',
+    'MediaStreamTrackAudioSourceNode.h',
     'OscillatorNode.h',
     'PannerNode.h',
     'PeriodicWave.h',
     'ScriptProcessorNode.h',
     'StereoPannerNode.h',
     'WaveShaperNode.h',
 ]
 
@@ -97,16 +98,17 @@ UNIFIED_SOURCES += [
     'DynamicsCompressorNode.cpp',
     'FFTBlock.cpp',
     'GainNode.cpp',
     'IIRFilterNode.cpp',
     'MediaBufferDecoder.cpp',
     'MediaElementAudioSourceNode.cpp',
     'MediaStreamAudioDestinationNode.cpp',
     'MediaStreamAudioSourceNode.cpp',
+    'MediaStreamTrackAudioSourceNode.cpp',
     'OscillatorNode.cpp',
     'PannerNode.cpp',
     'PeriodicWave.cpp',
     'ScriptProcessorNode.cpp',
     'StereoPannerNode.cpp',
     'ThreeDPoint.cpp',
     'WaveShaperNode.cpp',
     'WebAudioUtils.cpp',
--- a/dom/media/webaudio/test/mochitest.ini
+++ b/dom/media/webaudio/test/mochitest.ini
@@ -171,16 +171,18 @@ skip-if = toolkit == 'android' # bug 114
 [test_mediaStreamAudioDestinationNode.html]
 [test_mediaStreamAudioSourceNode.html]
 [test_mediaStreamAudioSourceNodeCrossOrigin.html]
 tags=capturestream
 [test_mediaStreamAudioSourceNodeNoGC.html]
 [test_mediaStreamAudioSourceNodePassThrough.html]
 [test_mediaStreamAudioSourceNodeResampling.html]
 tags=capturestream
+[test_mediaStreamTrackAudioSourceNode.html]
+[test_mediaStreamTrackAudioSourceNodeVideo.html]
 [test_mixingRules.html]
 skip-if = toolkit == 'android' # bug 1091965
 [test_nodeToParamConnection.html]
 [test_nodeCreationDocumentGone.html]
 [test_OfflineAudioContext.html]
 [test_offlineDestinationChannelCountLess.html]
 [test_offlineDestinationChannelCountMore.html]
 [test_oscillatorNode.html]
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNode.html
@@ -0,0 +1,54 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test MediaStreamTrackAudioSourceNode processing is correct</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="webaudio.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+function createBuffer(context) {
+  var buffer = context.createBuffer(2, 2048, context.sampleRate);
+  for (var i = 0; i < 2048; ++i) {
+    buffer.getChannelData(0)[i] = Math.sin(440 * 2 * Math.PI * i / context.sampleRate);
+    buffer.getChannelData(1)[i] = -buffer.getChannelData(0)[i];
+  }
+  return buffer;
+}
+
+var gTest = {
+  length: 2048,
+  skipOfflineContextTests: true,
+  createGraph: function(context) {
+    var sourceGraph = new AudioContext();
+    var source = sourceGraph.createBufferSource();
+    source.buffer = createBuffer(context);
+    var dest = sourceGraph.createMediaStreamDestination();
+    source.connect(dest);
+
+    // Extract first audio track from dest.stream
+    var track = dest.stream.getAudioTracks()[0];
+
+    source.start(0);
+
+    var mediaStreamTrackSource = new MediaStreamTrackAudioSourceNode(context, { mediaStreamTrack: track });
+    // channelCount and channelCountMode should have no effect
+    mediaStreamTrackSource.channelCount = 1;
+    mediaStreamTrackSource.channelCountMode = "explicit";
+    return mediaStreamTrackSource;
+  },
+  createExpectedBuffers: function(context) {
+    return createBuffer(context);
+  },
+};
+
+runTest();
+
+</script>
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNodeVideo.html
@@ -0,0 +1,31 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test MediaStreamTrackAudioSourceNode throw video track</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="/tests/dom/media/webaudio/test/webaudio.js"></script>
+  <script type="text/javascript" src="/tests/dom/media/tests/mochitest/head.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+  createVideoTrack = () => {
+    let c = document.createElement("canvas");
+    c.getContext("2d");
+    return c.captureStream().getTracks()[0];
+  }
+
+  let context = new AudioContext();
+  let track = createVideoTrack();
+
+  expectException(() => {
+    let mediaStreamTrackSource = new MediaStreamTrackAudioSourceNode(
+      context,
+      { mediaStreamTrack: track });
+  }, DOMException.INVALID_STATE_ERR);
+</script>
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/dom/webidl/MediaStreamTrackAudioSourceNode.webidl
@@ -0,0 +1,24 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://webaudio.github.io/web-audio-api/
+ *
+ * Copyright © 2012 W3C® (MIT, ERCIM, Keio), All Rights Reserved. W3C
+ * liability, trademark and document use rules apply.
+ */
+
+dictionary MediaStreamTrackAudioSourceOptions {
+    required MediaStreamTrack mediaStreamTrack;
+};
+
+[Pref="dom.webaudio.enabled",
+ Constructor(AudioContext context, MediaStreamTrackAudioSourceOptions options)]
+interface MediaStreamTrackAudioSourceNode : AudioNode {
+
+};
+
+// Mozilla extensions
+MediaStreamTrackAudioSourceNode implements AudioNodePassThrough;
--- a/dom/webidl/moz.build
+++ b/dom/webidl/moz.build
@@ -188,16 +188,19 @@ with Files("MediaDevice*"):
     BUG_COMPONENT = ("Core", "WebRTC")
 
 with Files("Media*Source*"):
     BUG_COMPONENT = ("Core", "Web Audio")
 
 with Files("MediaStream*"):
     BUG_COMPONENT = ("Core", "WebRTC")
 
+with Files("MediaStreamTrackAudio*"):
+    BUG_COMPONENT = ("Core", "Web Audio")
+
 with Files("MediaStreamAudio*"):
     BUG_COMPONENT = ("Core", "Web Audio")
 
 with Files("MediaEncryptedEvent.webidl"):
     BUG_COMPONENT = ("Core", "Audio/Video")
 
 with Files("MediaKey*"):
     BUG_COMPONENT = ("Core", "Audio/Video: Playback")
@@ -687,16 +690,17 @@ WEBIDL_FILES = [
     'MediaQueryList.webidl',
     'MediaRecorder.webidl',
     'MediaSource.webidl',
     'MediaStream.webidl',
     'MediaStreamAudioDestinationNode.webidl',
     'MediaStreamAudioSourceNode.webidl',
     'MediaStreamError.webidl',
     'MediaStreamTrack.webidl',
+    'MediaStreamTrackAudioSourceNode.webidl',
     'MediaTrackConstraintSet.webidl',
     'MediaTrackSettings.webidl',
     'MediaTrackSupportedConstraints.webidl',
     'MenuBoxObject.webidl',
     'MessageChannel.webidl',
     'MessageEvent.webidl',
     'MessagePort.webidl',
     'MimeType.webidl',