Bug 1324548 - Add MediaStreamTrackAudioSourceNode. r?padenot,pehrsons,baku
MozReview-Commit-ID: IdVqfNigMyu
--- a/dom/locales/en-US/chrome/dom/dom.properties
+++ b/dom/locales/en-US/chrome/dom/dom.properties
@@ -97,16 +97,18 @@ MediaDecodeAudioDataUnknownError=An unkn
# LOCALIZATION NOTE: Do not translate decodeAudioData.
MediaDecodeAudioDataInvalidContent=The buffer passed to decodeAudioData contains invalid content which cannot be decoded successfully.
# LOCALIZATION NOTE: Do not translate decodeAudioData.
MediaDecodeAudioDataNoAudio=The buffer passed to decodeAudioData does not contain any audio.
# LOCALIZATION NOTE: Do not translate HTMLMediaElement and createMediaElementSource.
MediaElementAudioSourceNodeCrossOrigin=The HTMLMediaElement passed to createMediaElementSource has a cross-origin resource, the node will output silence.
# LOCALIZATION NOTE: Do not translate MediaStream and createMediaStreamSource.
MediaStreamAudioSourceNodeCrossOrigin=The MediaStream passed to createMediaStreamSource has a cross-origin resource, the node will output silence.
+# LOCALIZATION NOTE : Do not translate MediaStreamTrack and createMediaStreamTrackSource.
+MediaStreamTrackAudioSourceNodeCrossOrigin=The MediaStreamTrack passed to createMediaStreamTrackSource is a cross-origin resource, the node will output slence.
MediaLoadExhaustedCandidates=All candidate resources failed to load. Media load paused.
MediaLoadSourceMissingSrc=<source> element has no “src” attribute. Media resource load failed.
# LOCALIZATION NOTE: %1$S is the Http error code the server returned (e.g. 404, 500, etc), %2$S is the URL of the media resource which failed to load.
MediaLoadHttpError=HTTP load failed with status %1$S. Load of media resource %2$S failed.
# LOCALIZATION NOTE: %S is the URL of the media resource which failed to load.
MediaLoadInvalidURI=Invalid URI. Load of media resource %S failed.
# LOCALIZATION NOTE: %1$S is the media resource's format/codec type (basically equivalent to the file type, e.g. MP4,AVI,WMV,MOV etc), %2$S is the URL of the media resource which failed to load.
MediaLoadUnsupportedTypeAttribute=Specified “type” attribute of “%1$S” is not supported. Load of media resource %2$S failed.
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/MediaStreamTrackAudioSourceNode.cpp
@@ -0,0 +1,199 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamTrackAudioSourceNode.h"
+#include "mozilla/dom/MediaStreamTrackAudioSourceNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeExternalInputStream.h"
+#include "AudioStreamTrack.h"
+#include "nsIDocument.h"
+#include "mozilla/CORSMode.h"
+#include "nsContentUtils.h"
+#include "nsIScriptError.h"
+
+namespace mozilla {
+namespace dom {
+
+NS_IMPL_CYCLE_COLLECTION_CLASS(MediaStreamTrackAudioSourceNode)
+
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(MediaStreamTrackAudioSourceNode)
+ tmp->Destroy();
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(AudioNode)
+
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mInputTrack)
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(MediaStreamTrackAudioSourceNode)
+NS_INTERFACE_MAP_END_INHERITING(AudioNode)
+
+NS_IMPL_ADDREF_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+NS_IMPL_RELEASE_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+
+MediaStreamTrackAudioSourceNode::MediaStreamTrackAudioSourceNode(AudioContext* aContext)
+ : AudioNode(aContext,
+ 2,
+ ChannelCountMode::Max,
+ ChannelInterpretation::Speakers)
+ , mTrackListener(this)
+{
+}
+
+/* static */ already_AddRefed<MediaStreamTrackAudioSourceNode>
+MediaStreamTrackAudioSourceNode::Create(AudioContext& aAudioContext,
+ const MediaStreamTrackAudioSourceOptions& aOptions,
+ ErrorResult& aRv)
+{
+ if (aAudioContext.IsOffline()) {
+ aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ return nullptr;
+ }
+
+ if (aAudioContext.CheckClosed(aRv)) {
+ return nullptr;
+ }
+
+ RefPtr<MediaStreamTrackAudioSourceNode> node =
+ new MediaStreamTrackAudioSourceNode(&aAudioContext);
+
+ node->Init(aOptions.mMediaStreamTrack, aRv);
+ if (aRv.Failed()) {
+ return nullptr;
+ }
+
+ return node.forget();
+}
+
+void
+MediaStreamTrackAudioSourceNode::Init(MediaStreamTrack* aMediaStreamTrack, ErrorResult& aRv)
+{
+ if (!aMediaStreamTrack) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return;
+ }
+
+ if (!aMediaStreamTrack->AsAudioStreamTrack()) {
+ aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
+ return;
+ }
+
+ MediaStreamGraph* graph = Context()->Graph();
+
+ AudioNodeEngine* engine = new MediaStreamTrackAudioSourceNodeEngine(this);
+ mStream = AudioNodeExternalInputStream::Create(graph, engine);
+
+ if (!mStream) {
+ return;
+ }
+
+ mInputTrack = aMediaStreamTrack;
+ ProcessedMediaStream* outputStream =
+ static_cast<ProcessedMediaStream*>(mStream.get());
+ mInputPort = mInputTrack->ForwardTrackContentsTo(outputStream);
+ PrincipalChanged(mInputTrack); // trigger enabling/disabling of the connector
+ mInputTrack->AddPrincipalChangeObserver(this);
+
+ mInputTrack->AddConsumer(&mTrackListener);
+
+}
+
+void
+MediaStreamTrackAudioSourceNode::Destroy()
+{
+ if (mInputTrack) {
+ mInputTrack->RemovePrincipalChangeObserver(this);
+ mInputTrack = nullptr;
+ }
+ if (mInputPort) {
+ mInputPort->Destroy();
+ mInputPort = nullptr;
+ }
+}
+
+MediaStreamTrackAudioSourceNode::~MediaStreamTrackAudioSourceNode()
+{
+ Destroy();
+}
+
+/**
+ * Changes the principal. Note that this will be called on the main thread, but
+ * changes will be enacted on the MediaStreamGraph thread. If the principal
+ * change results in the document principal losing access to the stream, then
+ * there needs to be other measures in place to ensure that any media that is
+ * governed by the new stream principal is not available to the MediaStreamGraph
+ * before this change completes. Otherwise, a site could get access to
+ * media that they are not authorized to receive.
+ *
+ * One solution is to block the altered content, call this method, then dispatch
+ * another change request to the MediaStreamGraph thread that allows the content
+ * under the new principal to flow. This might be unnecessary if the principal
+ * change is changing to be the document principal.
+ */
+void
+MediaStreamTrackAudioSourceNode::PrincipalChanged(MediaStreamTrack* aMediaStreamTrack)
+{
+ MOZ_ASSERT(aMediaStreamTrack == mInputTrack);
+
+ bool subsumes = false;
+ nsIDocument* doc = nullptr;
+ if (nsPIDOMWindowInner* parent = Context()->GetParentObject()) {
+ doc = parent->GetExtantDoc();
+ if (doc) {
+ nsIPrincipal* docPrincipal = doc->NodePrincipal();
+ nsIPrincipal* trackPrincipal = aMediaStreamTrack->GetPrincipal();
+ if (!trackPrincipal || NS_FAILED(docPrincipal->Subsumes(trackPrincipal, &subsumes))) {
+ subsumes = false;
+ }
+ }
+ }
+ auto stream = static_cast<AudioNodeExternalInputStream*>(mStream.get());
+ bool enabled = subsumes || aMediaStreamTrack->GetCORSMode() != CORS_NONE;
+ stream->SetInt32Parameter(MediaStreamTrackAudioSourceNodeEngine::ENABLE, enabled);
+
+ if (!enabled && doc) {
+ nsContentUtils::ReportToConsole(nsIScriptError::warningFlag,
+ NS_LITERAL_CSTRING("Web Audio"),
+ doc,
+ nsContentUtils::eDOM_PROPERTIES,
+ CrossOriginErrorString());
+ }
+}
+
+size_t
+MediaStreamTrackAudioSourceNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
+{
+ size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf);
+ if (mInputPort) {
+ amount += mInputPort->SizeOfIncludingThis(aMallocSizeOf);
+ }
+ return amount;
+}
+
+size_t
+MediaStreamTrackAudioSourceNode::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
+{
+ return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+
+void
+MediaStreamTrackAudioSourceNode::DestroyMediaStream()
+{
+ if (mInputPort) {
+ mInputPort->Destroy();
+ mInputPort = nullptr;
+ }
+ AudioNode::DestroyMediaStream();
+}
+
+JSObject*
+MediaStreamTrackAudioSourceNode::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
+{
+ return MediaStreamTrackAudioSourceNodeBinding::Wrap(aCx, this, aGivenProto);
+}
+
+} // namespace dom
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/MediaStreamTrackAudioSourceNode.h
@@ -0,0 +1,119 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaStreamTrackAudioSourceNode_h_
+#define MediaStreamTrackAudioSourceNode_h_
+
+#include "AudioNode.h"
+#include "AudioNodeEngine.h"
+
+namespace mozilla {
+
+namespace dom {
+
+class AudioContext;
+struct MediaStreamTrackAudioSourceOptions;
+
+class MediaStreamTrackAudioSourceNodeEngine final : public AudioNodeEngine
+{
+public:
+ explicit MediaStreamTrackAudioSourceNodeEngine(AudioNode* aNode)
+ : AudioNodeEngine(aNode), mEnabled(false) {}
+
+ bool IsEnabled() const { return mEnabled; }
+ enum Parameters {
+ ENABLE
+ };
+ void SetInt32Parameter(uint32_t aIndex, int32_t aValue) override
+ {
+ switch (aIndex) {
+ case ENABLE:
+ mEnabled = !!aValue;
+ break;
+ default:
+ NS_ERROR("MediaStreamTrackAudioSourceNodeEngine bad parameter index");
+ }
+ }
+
+private:
+ bool mEnabled;
+};
+
+class MediaStreamTrackAudioSourceNode
+ : public AudioNode
+ , public PrincipalChangeObserver<MediaStreamTrack>
+{
+public:
+ static already_AddRefed<MediaStreamTrackAudioSourceNode>
+ Create(AudioContext& aContext, const MediaStreamTrackAudioSourceOptions& aOptions,
+ ErrorResult& aRv);
+
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrackAudioSourceNode, AudioNode)
+
+ static already_AddRefed<MediaStreamTrackAudioSourceNode>
+ Constructor(const GlobalObject& aGlobal, AudioContext& aAudioContext,
+ const MediaStreamTrackAudioSourceOptions& aOptions, ErrorResult& aRv)
+ {
+ return Create(aAudioContext, aOptions, aRv);
+ }
+
+ JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
+
+ void DestroyMediaStream() override;
+
+ uint16_t NumberOfInputs() const override { return 0; }
+
+ const char* NodeType() const override
+ {
+ return "MediaStreamTrackAudioSourceNode";
+ }
+
+ virtual const char* CrossOriginErrorString() const
+ {
+ return "MediaStreamTrackAudioSourceNodeCrossOrigin";
+ }
+
+ size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
+ size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override;
+
+ // From PrincipalChangeObserver<MediaStreamTrack>.
+ void PrincipalChanged(MediaStreamTrack* aMediaStreamTrack) override;
+
+protected:
+ explicit MediaStreamTrackAudioSourceNode(AudioContext* aContext);
+ void Init(MediaStreamTrack* aMediaStreamTrack, ErrorResult& aRv);
+ void Destroy();
+ virtual ~MediaStreamTrackAudioSourceNode();
+
+
+ class TrackListener : public MediaStreamTrackConsumer
+ {
+ public:
+ TrackListener(MediaStreamTrackAudioSourceNode* aNode)
+ {
+ mNode = aNode;
+ }
+
+ void NotifyEnded(MediaStreamTrack* aTrack) override
+ {
+ mNode->MarkInactive();
+ }
+
+ private:
+ RefPtr<MediaStreamTrackAudioSourceNode> mNode;
+ };
+
+private:
+ RefPtr<MediaInputPort> mInputPort;
+ RefPtr<MediaStreamTrack> mInputTrack;
+ TrackListener mTrackListener;
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif
--- a/dom/media/webaudio/moz.build
+++ b/dom/media/webaudio/moz.build
@@ -58,16 +58,17 @@ EXPORTS.mozilla.dom += [
'ConvolverNode.h',
'DelayNode.h',
'DynamicsCompressorNode.h',
'GainNode.h',
'IIRFilterNode.h',
'MediaElementAudioSourceNode.h',
'MediaStreamAudioDestinationNode.h',
'MediaStreamAudioSourceNode.h',
+ 'MediaStreamTrackAudioSourceNode.h',
'OscillatorNode.h',
'PannerNode.h',
'PeriodicWave.h',
'ScriptProcessorNode.h',
'StereoPannerNode.h',
'WaveShaperNode.h',
]
@@ -97,16 +98,17 @@ UNIFIED_SOURCES += [
'DynamicsCompressorNode.cpp',
'FFTBlock.cpp',
'GainNode.cpp',
'IIRFilterNode.cpp',
'MediaBufferDecoder.cpp',
'MediaElementAudioSourceNode.cpp',
'MediaStreamAudioDestinationNode.cpp',
'MediaStreamAudioSourceNode.cpp',
+ 'MediaStreamTrackAudioSourceNode.cpp',
'OscillatorNode.cpp',
'PannerNode.cpp',
'PeriodicWave.cpp',
'ScriptProcessorNode.cpp',
'StereoPannerNode.cpp',
'ThreeDPoint.cpp',
'WaveShaperNode.cpp',
'WebAudioUtils.cpp',
--- a/dom/media/webaudio/test/mochitest.ini
+++ b/dom/media/webaudio/test/mochitest.ini
@@ -171,16 +171,18 @@ skip-if = toolkit == 'android' # bug 114
[test_mediaStreamAudioDestinationNode.html]
[test_mediaStreamAudioSourceNode.html]
[test_mediaStreamAudioSourceNodeCrossOrigin.html]
tags=capturestream
[test_mediaStreamAudioSourceNodeNoGC.html]
[test_mediaStreamAudioSourceNodePassThrough.html]
[test_mediaStreamAudioSourceNodeResampling.html]
tags=capturestream
+[test_mediaStreamTrackAudioSourceNode.html]
+[test_mediaStreamTrackAudioSourceNodeVideo.html]
[test_mixingRules.html]
skip-if = toolkit == 'android' # bug 1091965
[test_nodeToParamConnection.html]
[test_nodeCreationDocumentGone.html]
[test_OfflineAudioContext.html]
[test_offlineDestinationChannelCountLess.html]
[test_offlineDestinationChannelCountMore.html]
[test_oscillatorNode.html]
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNode.html
@@ -0,0 +1,54 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+ <title>Test MediaStreamTrackAudioSourceNode processing is correct</title>
+ <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <script type="text/javascript" src="webaudio.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+function createBuffer(context) {
+ var buffer = context.createBuffer(2, 2048, context.sampleRate);
+ for (var i = 0; i < 2048; ++i) {
+ buffer.getChannelData(0)[i] = Math.sin(440 * 2 * Math.PI * i / context.sampleRate);
+ buffer.getChannelData(1)[i] = -buffer.getChannelData(0)[i];
+ }
+ return buffer;
+}
+
+var gTest = {
+ length: 2048,
+ skipOfflineContextTests: true,
+ createGraph: function(context) {
+ var sourceGraph = new AudioContext();
+ var source = sourceGraph.createBufferSource();
+ source.buffer = createBuffer(context);
+ var dest = sourceGraph.createMediaStreamDestination();
+ source.connect(dest);
+
+ // Extract first audio track from dest.stream
+ var track = dest.stream.getAudioTracks()[0];
+
+ source.start(0);
+
+ var mediaStreamTrackSource = new MediaStreamTrackAudioSourceNode(context, { mediaStreamTrack: track });
+ // channelCount and channelCountMode should have no effect
+ mediaStreamTrackSource.channelCount = 1;
+ mediaStreamTrackSource.channelCountMode = "explicit";
+ return mediaStreamTrackSource;
+ },
+ createExpectedBuffers: function(context) {
+ return createBuffer(context);
+ },
+};
+
+runTest();
+
+</script>
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/dom/media/webaudio/test/test_mediaStreamTrackAudioSourceNodeVideo.html
@@ -0,0 +1,28 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+ <title>Test MediaStreamTrackAudioSourceNode throw video track</title>
+ <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+ <script type="text/javascript" src="/tests/dom/media/webaudio/test/webaudio.js"></script>
+ <script type="text/javascript" src="/tests/dom/media/tests/mochitest/head.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+ let context = new AudioContext();
+ let canvas = document.createElement("canvas");
+ canvas.getContext("2d");
+ let track = canvas.captureStream().getTracks()[0];
+
+ expectException(() => {
+ let mediaStreamTrackSource = new MediaStreamTrackAudioSourceNode(
+ context,
+ { mediaStreamTrack: track });
+ }, DOMException.INVALID_STATE_ERR);
+</script>
+</pre>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/dom/webidl/MediaStreamTrackAudioSourceNode.webidl
@@ -0,0 +1,24 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://webaudio.github.io/web-audio-api/
+ *
+ * Copyright © 2012 W3C® (MIT, ERCIM, Keio), All Rights Reserved. W3C
+ * liability, trademark and document use rules apply.
+ */
+
+dictionary MediaStreamTrackAudioSourceOptions {
+ required MediaStreamTrack mediaStreamTrack;
+};
+
+[Pref="dom.webaudio.enabled",
+ Constructor(AudioContext context, MediaStreamTrackAudioSourceOptions options)]
+interface MediaStreamTrackAudioSourceNode : AudioNode {
+
+};
+
+// Mozilla extensions
+MediaStreamTrackAudioSourceNode implements AudioNodePassThrough;
--- a/dom/webidl/moz.build
+++ b/dom/webidl/moz.build
@@ -185,16 +185,19 @@ with Files("MediaDevice*"):
BUG_COMPONENT = ("Core", "WebRTC")
with Files("Media*Source*"):
BUG_COMPONENT = ("Core", "Web Audio")
with Files("MediaStream*"):
BUG_COMPONENT = ("Core", "WebRTC")
+with Files("MediaStreamTrackAudio*"):
+ BUG_COMPONENT = ("Core", "Web Audio")
+
with Files("MediaStreamAudio*"):
BUG_COMPONENT = ("Core", "Web Audio")
with Files("MediaEncryptedEvent.webidl"):
BUG_COMPONENT = ("Core", "Audio/Video")
with Files("MediaKey*"):
BUG_COMPONENT = ("Core", "Audio/Video: Playback")
@@ -673,16 +676,17 @@ WEBIDL_FILES = [
'MediaQueryList.webidl',
'MediaRecorder.webidl',
'MediaSource.webidl',
'MediaStream.webidl',
'MediaStreamAudioDestinationNode.webidl',
'MediaStreamAudioSourceNode.webidl',
'MediaStreamError.webidl',
'MediaStreamTrack.webidl',
+ 'MediaStreamTrackAudioSourceNode.webidl',
'MediaTrackConstraintSet.webidl',
'MediaTrackSettings.webidl',
'MediaTrackSupportedConstraints.webidl',
'MenuBoxObject.webidl',
'MessageChannel.webidl',
'MessageEvent.webidl',
'MessagePort.webidl',
'MimeType.webidl',