1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
6
7 #include "AudioWorkletNode.h"
8
9 #include "AudioNodeEngine.h"
10 #include "AudioParamMap.h"
11 #include "AudioWorkletImpl.h"
12 #include "js/Array.h" // JS::{Get,Set}ArrayLength, JS::NewArrayLength
13 #include "js/Exception.h"
14 #include "js/experimental/TypedData.h" // JS_NewFloat32Array, JS_GetFloat32ArrayData, JS_GetTypedArrayLength, JS_GetArrayBufferViewBuffer
15 #include "mozilla/dom/AudioWorkletNodeBinding.h"
16 #include "mozilla/dom/AudioParamMapBinding.h"
17 #include "mozilla/dom/AutoEntryScript.h"
18 #include "mozilla/dom/RootedDictionary.h"
19 #include "mozilla/dom/ErrorEvent.h"
20 #include "mozilla/dom/Worklet.h"
21 #include "nsIScriptGlobalObject.h"
22 #include "AudioParam.h"
23 #include "AudioDestinationNode.h"
24 #include "mozilla/dom/MessageChannel.h"
25 #include "mozilla/dom/MessagePort.h"
26 #include "mozilla/ScopeExit.h"
27 #include "nsReadableUtils.h"
28 #include "mozilla/Span.h"
29 #include "PlayingRefChangeHandler.h"
30 #include "nsPrintfCString.h"
31 #include "Tracing.h"
32
33 namespace mozilla::dom {
34
35 NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(AudioWorkletNode, AudioNode)
36 NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioWorkletNode, AudioNode, mPort,
37 mParameters)
38
39 struct NamedAudioParamTimeline {
NamedAudioParamTimelinemozilla::dom::NamedAudioParamTimeline40 explicit NamedAudioParamTimeline(const AudioParamDescriptor& aParamDescriptor)
41 : mName(aParamDescriptor.mName),
42 mTimeline(aParamDescriptor.mDefaultValue) {}
43 const nsString mName;
44 AudioParamTimeline mTimeline;
45 };
46
47 struct ProcessorErrorDetails {
ProcessorErrorDetailsmozilla::dom::ProcessorErrorDetails48 ProcessorErrorDetails() : mLineno(0), mColno(0) {}
49 unsigned mLineno;
50 unsigned mColno;
51 nsString mFilename;
52 nsString mMessage;
53 };
54
55 class WorkletNodeEngine final : public AudioNodeEngine {
56 public:
WorkletNodeEngine(AudioWorkletNode * aNode,AudioDestinationNode * aDestinationNode,nsTArray<NamedAudioParamTimeline> && aParamTimelines,const Optional<Sequence<uint32_t>> & aOutputChannelCount)57 WorkletNodeEngine(AudioWorkletNode* aNode,
58 AudioDestinationNode* aDestinationNode,
59 nsTArray<NamedAudioParamTimeline>&& aParamTimelines,
60 const Optional<Sequence<uint32_t>>& aOutputChannelCount)
61 : AudioNodeEngine(aNode),
62 mDestination(aDestinationNode->Track()),
63 mParamTimelines(std::move(aParamTimelines)) {
64 if (aOutputChannelCount.WasPassed()) {
65 mOutputChannelCount = aOutputChannelCount.Value();
66 }
67 }
68
69 MOZ_CAN_RUN_SCRIPT
70 void ConstructProcessor(AudioWorkletImpl* aWorkletImpl,
71 const nsAString& aName,
72 NotNull<StructuredCloneHolder*> aSerializedOptions,
73 UniqueMessagePortId& aPortIdentifier,
74 AudioNodeTrack* aTrack);
75
RecvTimelineEvent(uint32_t aIndex,AudioTimelineEvent & aEvent)76 void RecvTimelineEvent(uint32_t aIndex, AudioTimelineEvent& aEvent) override {
77 MOZ_ASSERT(mDestination);
78 WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, mDestination);
79
80 if (aIndex < mParamTimelines.Length()) {
81 mParamTimelines[aIndex].mTimeline.InsertEvent<int64_t>(aEvent);
82 } else {
83 NS_ERROR("Bad WorkletNodeEngine timeline event index");
84 }
85 }
86
ProcessBlock(AudioNodeTrack * aTrack,GraphTime aFrom,const AudioBlock & aInput,AudioBlock * aOutput,bool * aFinished)87 void ProcessBlock(AudioNodeTrack* aTrack, GraphTime aFrom,
88 const AudioBlock& aInput, AudioBlock* aOutput,
89 bool* aFinished) override {
90 MOZ_ASSERT(InputCount() <= 1);
91 MOZ_ASSERT(OutputCount() <= 1);
92 ProcessBlocksOnPorts(aTrack, aFrom, Span(&aInput, InputCount()),
93 Span(aOutput, OutputCount()), aFinished);
94 }
95
96 void ProcessBlocksOnPorts(AudioNodeTrack* aTrack, GraphTime aFrom,
97 Span<const AudioBlock> aInput,
98 Span<AudioBlock> aOutput, bool* aFinished) override;
99
OnGraphThreadDone()100 void OnGraphThreadDone() override { ReleaseJSResources(); }
101
IsActive() const102 bool IsActive() const override { return mKeepEngineActive; }
103
104 // Vector<T> supports non-memmovable types such as PersistentRooted
105 // (without any need to jump through hoops like
106 // MOZ_DECLARE_RELOCATE_USING_MOVE_CONSTRUCTOR_FOR_TEMPLATE for nsTArray).
107 // PersistentRooted is used because these AudioWorkletGlobalScope scope
108 // objects may be kept alive as long as the AudioWorkletNode in the
109 // main-thread global.
110 struct Channels {
111 Vector<JS::PersistentRooted<JSObject*>, GUESS_AUDIO_CHANNELS>
112 mFloat32Arrays;
113 JS::PersistentRooted<JSObject*> mJSArray;
114 // For SetArrayElements():
operator JS::Handle<JSObject*>mozilla::dom::WorkletNodeEngine::Channels115 operator JS::Handle<JSObject*>() const { return mJSArray; }
116 };
117 struct Ports {
118 Vector<Channels, 1> mPorts;
119 JS::PersistentRooted<JSObject*> mJSArray;
120 };
121 struct ParameterValues {
122 Vector<JS::PersistentRooted<JSObject*>> mFloat32Arrays;
123 JS::PersistentRooted<JSObject*> mJSObject;
124 };
125
126 private:
ParameterCount()127 size_t ParameterCount() { return mParamTimelines.Length(); }
128 void SendProcessorError(AudioNodeTrack* aTrack, JSContext* aCx);
129 bool CallProcess(AudioNodeTrack* aTrack, JSContext* aCx,
130 JS::Handle<JS::Value> aCallable);
131 void ProduceSilence(AudioNodeTrack* aTrack, Span<AudioBlock> aOutput);
132 void SendErrorToMainThread(AudioNodeTrack* aTrack,
133 const ProcessorErrorDetails& aDetails);
134
ReleaseJSResources()135 void ReleaseJSResources() {
136 mInputs.mPorts.clearAndFree();
137 mOutputs.mPorts.clearAndFree();
138 mParameters.mFloat32Arrays.clearAndFree();
139 mInputs.mJSArray.reset();
140 mOutputs.mJSArray.reset();
141 mParameters.mJSObject.reset();
142 mGlobal = nullptr;
143 // This is equivalent to setting [[callable process]] to false.
144 mProcessor.reset();
145 }
146
147 RefPtr<AudioNodeTrack> mDestination;
148 nsTArray<uint32_t> mOutputChannelCount;
149 nsTArray<NamedAudioParamTimeline> mParamTimelines;
150 // The AudioWorkletGlobalScope-associated objects referenced from
151 // WorkletNodeEngine are typically kept alive as long as the
152 // AudioWorkletNode in the main-thread global. The objects must be released
153 // on the rendering thread, which usually happens simply because
154 // AudioWorkletNode is such that the last AudioNodeTrack reference is
155 // released by the MTG. That occurs on the rendering thread except during
156 // process shutdown, in which case NotifyForcedShutdown() is called on the
157 // rendering thread.
158 //
159 // mInputs, mOutputs and mParameters keep references to all objects passed to
160 // process(), for reuse of the same objects. The JS objects are all in the
161 // compartment of the realm of mGlobal. Properties on the objects may be
162 // replaced by script, so don't assume that getting indexed properties on the
163 // JS arrays will return the same objects. Only objects and buffers created
164 // by the implementation are modified or read by the implementation.
165 Ports mInputs;
166 Ports mOutputs;
167 ParameterValues mParameters;
168
169 RefPtr<AudioWorkletGlobalScope> mGlobal;
170 JS::PersistentRooted<JSObject*> mProcessor;
171
172 // mProcessorIsActive is named [[active source]] in the spec.
173 // It is initially true and so at least the first process()
174 // call will not be skipped when there are no active inputs.
175 bool mProcessorIsActive = true;
176 // mKeepEngineActive ensures another call to ProcessBlocksOnPorts(), even if
177 // there are no active inputs. Its transitions to false lag those of
178 // mProcessorIsActive by one call to ProcessBlocksOnPorts() so that
179 // downstream engines can addref their nodes before this engine's node is
180 // released.
181 bool mKeepEngineActive = true;
182 };
183
SendErrorToMainThread(AudioNodeTrack * aTrack,const ProcessorErrorDetails & aDetails)184 void WorkletNodeEngine::SendErrorToMainThread(
185 AudioNodeTrack* aTrack, const ProcessorErrorDetails& aDetails) {
186 RefPtr<AudioNodeTrack> track = aTrack;
187 NS_DispatchToMainThread(NS_NewRunnableFunction(
188 "WorkletNodeEngine::SendProcessorError",
189 [track = std::move(track), aDetails]() mutable {
190 AudioWorkletNode* node =
191 static_cast<AudioWorkletNode*>(track->Engine()->NodeMainThread());
192 if (!node) {
193 return;
194 }
195 node->DispatchProcessorErrorEvent(aDetails);
196 }));
197 }
198
SendProcessorError(AudioNodeTrack * aTrack,JSContext * aCx)199 void WorkletNodeEngine::SendProcessorError(AudioNodeTrack* aTrack,
200 JSContext* aCx) {
201 // Note that once an exception is thrown, the processor will output silence
202 // throughout its lifetime.
203 ReleaseJSResources();
204 // The processor errored out while getting a context, try to tell the node
205 // anyways.
206 if (!aCx || !JS_IsExceptionPending(aCx)) {
207 ProcessorErrorDetails details;
208 details.mMessage.Assign(u"Unknown processor error");
209 SendErrorToMainThread(aTrack, details);
210 return;
211 }
212
213 JS::ExceptionStack exnStack(aCx);
214 if (JS::StealPendingExceptionStack(aCx, &exnStack)) {
215 JS::ErrorReportBuilder jsReport(aCx);
216 if (!jsReport.init(aCx, exnStack,
217 JS::ErrorReportBuilder::WithSideEffects)) {
218 ProcessorErrorDetails details;
219 details.mMessage.Assign(u"Unknown processor error");
220 SendErrorToMainThread(aTrack, details);
221 // Set the exception and stack back to have it in the console with a stack
222 // trace.
223 JS::SetPendingExceptionStack(aCx, exnStack);
224 return;
225 }
226
227 ProcessorErrorDetails details;
228
229 CopyUTF8toUTF16(mozilla::MakeStringSpan(jsReport.report()->filename),
230 details.mFilename);
231
232 xpc::ErrorReport::ErrorReportToMessageString(jsReport.report(),
233 details.mMessage);
234 details.mLineno = jsReport.report()->lineno;
235 details.mColno = jsReport.report()->column;
236 MOZ_ASSERT(!jsReport.report()->isMuted);
237
238 SendErrorToMainThread(aTrack, details);
239
240 // Set the exception and stack back to have it in the console with a stack
241 // trace.
242 JS::SetPendingExceptionStack(aCx, exnStack);
243 } else {
244 NS_WARNING("No exception, but processor errored out?");
245 }
246 }
247
ConstructProcessor(AudioWorkletImpl * aWorkletImpl,const nsAString & aName,NotNull<StructuredCloneHolder * > aSerializedOptions,UniqueMessagePortId & aPortIdentifier,AudioNodeTrack * aTrack)248 void WorkletNodeEngine::ConstructProcessor(
249 AudioWorkletImpl* aWorkletImpl, const nsAString& aName,
250 NotNull<StructuredCloneHolder*> aSerializedOptions,
251 UniqueMessagePortId& aPortIdentifier, AudioNodeTrack* aTrack) {
252 MOZ_ASSERT(mInputs.mPorts.empty() && mOutputs.mPorts.empty());
253 RefPtr<AudioWorkletGlobalScope> global = aWorkletImpl->GetGlobalScope();
254 if (!global) {
255 // A global was previously used to register this kind of processor. If it
256 // no longer exists now, that is because the document is going away and so
257 // there is no need to send an error.
258 return;
259 }
260 AutoJSAPI api;
261 if (NS_WARN_IF(!api.Init(global))) {
262 SendProcessorError(aTrack, nullptr);
263 return;
264 }
265 JSContext* cx = api.cx();
266 mProcessor.init(cx);
267 if (!global->ConstructProcessor(cx, aName, aSerializedOptions,
268 aPortIdentifier, &mProcessor) ||
269 // mInputs and mOutputs outer arrays are fixed length and so much of the
270 // initialization need only be performed once (i.e. here).
271 NS_WARN_IF(!mInputs.mPorts.growBy(InputCount())) ||
272 NS_WARN_IF(!mOutputs.mPorts.growBy(OutputCount()))) {
273 SendProcessorError(aTrack, cx);
274 return;
275 }
276 mGlobal = std::move(global);
277 mInputs.mJSArray.init(cx);
278 mOutputs.mJSArray.init(cx);
279 for (auto& port : mInputs.mPorts) {
280 port.mJSArray.init(cx);
281 }
282 for (auto& port : mOutputs.mPorts) {
283 port.mJSArray.init(cx);
284 }
285 JSObject* object = JS_NewPlainObject(cx);
286 if (NS_WARN_IF(!object)) {
287 SendProcessorError(aTrack, cx);
288 return;
289 }
290
291 mParameters.mJSObject.init(cx, object);
292 if (NS_WARN_IF(!mParameters.mFloat32Arrays.growBy(ParameterCount()))) {
293 SendProcessorError(aTrack, cx);
294 return;
295 }
296 for (size_t i = 0; i < mParamTimelines.Length(); i++) {
297 auto& float32ArraysRef = mParameters.mFloat32Arrays;
298 float32ArraysRef[i].init(cx);
299 JSObject* array = JS_NewFloat32Array(cx, WEBAUDIO_BLOCK_SIZE);
300 if (NS_WARN_IF(!array)) {
301 SendProcessorError(aTrack, cx);
302 return;
303 }
304
305 float32ArraysRef[i] = array;
306 if (NS_WARN_IF(!JS_DefineUCProperty(
307 cx, mParameters.mJSObject, mParamTimelines[i].mName.get(),
308 mParamTimelines[i].mName.Length(), float32ArraysRef[i],
309 JSPROP_ENUMERATE))) {
310 SendProcessorError(aTrack, cx);
311 return;
312 }
313 }
314 if (NS_WARN_IF(!JS_FreezeObject(cx, mParameters.mJSObject))) {
315 SendProcessorError(aTrack, cx);
316 return;
317 }
318 }
319
320 // Type T should support the length() and operator[]() methods and the return
321 // type of |operator[]() const| should support conversion to Handle<JSObject*>.
322 template <typename T>
SetArrayElements(JSContext * aCx,const T & aElements,JS::Handle<JSObject * > aArray)323 static bool SetArrayElements(JSContext* aCx, const T& aElements,
324 JS::Handle<JSObject*> aArray) {
325 for (size_t i = 0; i < aElements.length(); ++i) {
326 if (!JS_DefineElement(aCx, aArray, i, aElements[i], JSPROP_ENUMERATE)) {
327 return false;
328 }
329 }
330
331 return true;
332 }
333
334 template <typename T>
PrepareArray(JSContext * aCx,const T & aElements,JS::MutableHandle<JSObject * > aArray)335 static bool PrepareArray(JSContext* aCx, const T& aElements,
336 JS::MutableHandle<JSObject*> aArray) {
337 size_t length = aElements.length();
338 if (aArray) {
339 // Attempt to reuse.
340 uint32_t oldLength;
341 if (JS::GetArrayLength(aCx, aArray, &oldLength) &&
342 (oldLength == length || JS::SetArrayLength(aCx, aArray, length)) &&
343 SetArrayElements(aCx, aElements, aArray)) {
344 return true;
345 }
346 // Script may have frozen the array. Try again with a new Array.
347 JS_ClearPendingException(aCx);
348 }
349 JSObject* array = JS::NewArrayObject(aCx, length);
350 if (NS_WARN_IF(!array)) {
351 return false;
352 }
353 aArray.set(array);
354 return SetArrayElements(aCx, aElements, aArray);
355 }
356
357 enum class ArrayElementInit { None, Zero };
358
359 // Exactly when to create new Float32Array and Array objects is not specified.
360 // This approach aims to minimize garbage creation, while continuing to
361 // function after objects are modified by content.
362 // See https://github.com/WebAudio/web-audio-api/issues/1934 and
363 // https://github.com/WebAudio/web-audio-api/issues/1933
PrepareBufferArrays(JSContext * aCx,Span<const AudioBlock> aBlocks,WorkletNodeEngine::Ports * aPorts,ArrayElementInit aInit)364 static bool PrepareBufferArrays(JSContext* aCx, Span<const AudioBlock> aBlocks,
365 WorkletNodeEngine::Ports* aPorts,
366 ArrayElementInit aInit) {
367 MOZ_ASSERT(aBlocks.Length() == aPorts->mPorts.length());
368 for (size_t i = 0; i < aBlocks.Length(); ++i) {
369 size_t channelCount = aBlocks[i].ChannelCount();
370 WorkletNodeEngine::Channels& portRef = aPorts->mPorts[i];
371
372 auto& float32ArraysRef = portRef.mFloat32Arrays;
373 for (auto& channelRef : float32ArraysRef) {
374 size_t length = JS_GetTypedArrayLength(channelRef);
375 if (length != WEBAUDIO_BLOCK_SIZE) {
376 // Script has detached array buffers. Create new objects.
377 JSObject* array = JS_NewFloat32Array(aCx, WEBAUDIO_BLOCK_SIZE);
378 if (NS_WARN_IF(!array)) {
379 return false;
380 }
381 channelRef = array;
382 } else if (aInit == ArrayElementInit::Zero) {
383 // Need only zero existing arrays as new arrays are already zeroed.
384 JS::AutoCheckCannotGC nogc;
385 bool isShared;
386 float* elementData =
387 JS_GetFloat32ArrayData(channelRef, &isShared, nogc);
388 MOZ_ASSERT(!isShared); // Was created as unshared
389 std::fill_n(elementData, WEBAUDIO_BLOCK_SIZE, 0.0f);
390 }
391 }
392 // Enlarge if necessary...
393 if (NS_WARN_IF(!float32ArraysRef.reserve(channelCount))) {
394 return false;
395 }
396 while (float32ArraysRef.length() < channelCount) {
397 JSObject* array = JS_NewFloat32Array(aCx, WEBAUDIO_BLOCK_SIZE);
398 if (NS_WARN_IF(!array)) {
399 return false;
400 }
401 float32ArraysRef.infallibleEmplaceBack(aCx, array);
402 }
403 // ... or shrink if necessary.
404 float32ArraysRef.shrinkTo(channelCount);
405
406 if (NS_WARN_IF(!PrepareArray(aCx, float32ArraysRef, &portRef.mJSArray))) {
407 return false;
408 }
409 }
410
411 return !(NS_WARN_IF(!PrepareArray(aCx, aPorts->mPorts, &aPorts->mJSArray)));
412 }
413
414 // This runs JS script. MediaTrackGraph control messages, which would
415 // potentially destroy the WorkletNodeEngine and its AudioNodeTrack, cannot
416 // be triggered by script. They are not run from an nsIThread event loop and
417 // do not run until after ProcessBlocksOnPorts() has returned.
CallProcess(AudioNodeTrack * aTrack,JSContext * aCx,JS::Handle<JS::Value> aCallable)418 bool WorkletNodeEngine::CallProcess(AudioNodeTrack* aTrack, JSContext* aCx,
419 JS::Handle<JS::Value> aCallable) {
420 TRACE();
421
422 JS::RootedVector<JS::Value> argv(aCx);
423 if (NS_WARN_IF(!argv.resize(3))) {
424 return false;
425 }
426 argv[0].setObject(*mInputs.mJSArray);
427 argv[1].setObject(*mOutputs.mJSArray);
428 argv[2].setObject(*mParameters.mJSObject);
429 JS::Rooted<JS::Value> rval(aCx);
430 if (!JS::Call(aCx, mProcessor, aCallable, argv, &rval)) {
431 return false;
432 }
433
434 mProcessorIsActive = JS::ToBoolean(rval);
435 // Transitions of mProcessorIsActive to false do not trigger
436 // PlayingRefChangeHandler::RELEASE until silence is produced in the next
437 // block. This allows downstream engines receiving this non-silence block
438 // to take a reference to their nodes before this engine's node releases its
439 // down node references.
440 if (mProcessorIsActive && !mKeepEngineActive) {
441 mKeepEngineActive = true;
442 RefPtr<PlayingRefChangeHandler> refchanged =
443 new PlayingRefChangeHandler(aTrack, PlayingRefChangeHandler::ADDREF);
444 aTrack->Graph()->DispatchToMainThreadStableState(refchanged.forget());
445 }
446 return true;
447 }
448
ProduceSilence(AudioNodeTrack * aTrack,Span<AudioBlock> aOutput)449 void WorkletNodeEngine::ProduceSilence(AudioNodeTrack* aTrack,
450 Span<AudioBlock> aOutput) {
451 if (mKeepEngineActive) {
452 mKeepEngineActive = false;
453 aTrack->ScheduleCheckForInactive();
454 RefPtr<PlayingRefChangeHandler> refchanged =
455 new PlayingRefChangeHandler(aTrack, PlayingRefChangeHandler::RELEASE);
456 aTrack->Graph()->DispatchToMainThreadStableState(refchanged.forget());
457 }
458 for (AudioBlock& output : aOutput) {
459 output.SetNull(WEBAUDIO_BLOCK_SIZE);
460 }
461 }
462
ProcessBlocksOnPorts(AudioNodeTrack * aTrack,GraphTime aFrom,Span<const AudioBlock> aInput,Span<AudioBlock> aOutput,bool * aFinished)463 void WorkletNodeEngine::ProcessBlocksOnPorts(AudioNodeTrack* aTrack,
464 GraphTime aFrom,
465 Span<const AudioBlock> aInput,
466 Span<AudioBlock> aOutput,
467 bool* aFinished) {
468 MOZ_ASSERT(aInput.Length() == InputCount());
469 MOZ_ASSERT(aOutput.Length() == OutputCount());
470 TRACE();
471
472 bool isSilent = true;
473 if (mProcessor) {
474 if (mProcessorIsActive) {
475 isSilent = false; // call process()
476 } else { // [[active source]] is false.
477 // Call process() only if an input is actively processing.
478 for (const AudioBlock& input : aInput) {
479 if (!input.IsNull()) {
480 isSilent = false;
481 break;
482 }
483 }
484 }
485 }
486 if (isSilent) {
487 ProduceSilence(aTrack, aOutput);
488 return;
489 }
490
491 if (!mOutputChannelCount.IsEmpty()) {
492 MOZ_ASSERT(mOutputChannelCount.Length() == aOutput.Length());
493 for (size_t o = 0; o < aOutput.Length(); ++o) {
494 aOutput[o].AllocateChannels(mOutputChannelCount[o]);
495 }
496 } else if (aInput.Length() == 1 && aOutput.Length() == 1) {
497 uint32_t channelCount = std::max(aInput[0].ChannelCount(), 1U);
498 aOutput[0].AllocateChannels(channelCount);
499 } else {
500 for (AudioBlock& output : aOutput) {
501 output.AllocateChannels(1);
502 }
503 }
504
505 AutoEntryScript aes(mGlobal, "Worklet Process");
506 JSContext* cx = aes.cx();
507 auto produceSilenceWithError = MakeScopeExit([this, aTrack, cx, &aOutput] {
508 SendProcessorError(aTrack, cx);
509 ProduceSilence(aTrack, aOutput);
510 });
511
512 JS::Rooted<JS::Value> process(cx);
513 if (!JS_GetProperty(cx, mProcessor, "process", &process) ||
514 !process.isObject() || !JS::IsCallable(&process.toObject()) ||
515 !PrepareBufferArrays(cx, aInput, &mInputs, ArrayElementInit::None) ||
516 !PrepareBufferArrays(cx, aOutput, &mOutputs, ArrayElementInit::Zero)) {
517 // process() not callable or OOM.
518 return;
519 }
520
521 // Copy input values to JS objects.
522 for (size_t i = 0; i < aInput.Length(); ++i) {
523 const AudioBlock& input = aInput[i];
524 size_t channelCount = input.ChannelCount();
525 if (channelCount == 0) {
526 // Null blocks have AUDIO_FORMAT_SILENCE.
527 // Don't call ChannelData<float>().
528 continue;
529 }
530 float volume = input.mVolume;
531 const auto& channelData = input.ChannelData<float>();
532 const auto& float32Arrays = mInputs.mPorts[i].mFloat32Arrays;
533 JS::AutoCheckCannotGC nogc;
534 for (size_t c = 0; c < channelCount; ++c) {
535 bool isShared;
536 float* dest = JS_GetFloat32ArrayData(float32Arrays[c], &isShared, nogc);
537 MOZ_ASSERT(!isShared); // Was created as unshared
538 AudioBlockCopyChannelWithScale(channelData[c], volume, dest);
539 }
540 }
541
542 TrackTime tick = mDestination->GraphTimeToTrackTime(aFrom);
543 // Compute and copy parameter values to JS objects.
544 for (size_t i = 0; i < mParamTimelines.Length(); ++i) {
545 const auto& float32Arrays = mParameters.mFloat32Arrays[i];
546 size_t length = JS_GetTypedArrayLength(float32Arrays);
547
548 // If the Float32Array that is supposed to hold the values for a particular
549 // AudioParam has been detached, error out. This is being worked on in
550 // https://github.com/WebAudio/web-audio-api/issues/1933 and
551 // https://bugzilla.mozilla.org/show_bug.cgi?id=1619486
552 if (length != WEBAUDIO_BLOCK_SIZE) {
553 return;
554 }
555 JS::AutoCheckCannotGC nogc;
556 bool isShared;
557 float* dest = JS_GetFloat32ArrayData(float32Arrays, &isShared, nogc);
558 MOZ_ASSERT(!isShared); // Was created as unshared
559
560 size_t frames =
561 mParamTimelines[i].mTimeline.HasSimpleValue() ? 1 : WEBAUDIO_BLOCK_SIZE;
562 mParamTimelines[i].mTimeline.GetValuesAtTime(tick, dest, frames);
563 // https://bugzilla.mozilla.org/show_bug.cgi?id=1616599
564 if (frames == 1) {
565 std::fill_n(dest + 1, WEBAUDIO_BLOCK_SIZE - 1, dest[0]);
566 }
567 }
568
569 if (!CallProcess(aTrack, cx, process)) {
570 // An exception occurred.
571 /**
572 * https://webaudio.github.io/web-audio-api/#dom-audioworkletnode-onprocessorerror
573 * Note that once an exception is thrown, the processor will output silence
574 * throughout its lifetime.
575 */
576 return;
577 }
578
579 // Copy output values from JS objects.
580 for (size_t o = 0; o < aOutput.Length(); ++o) {
581 AudioBlock* output = &aOutput[o];
582 size_t channelCount = output->ChannelCount();
583 const auto& float32Arrays = mOutputs.mPorts[o].mFloat32Arrays;
584 for (size_t c = 0; c < channelCount; ++c) {
585 size_t length = JS_GetTypedArrayLength(float32Arrays[c]);
586 if (length != WEBAUDIO_BLOCK_SIZE) {
587 // ArrayBuffer has been detached. Behavior is unspecified.
588 // https://github.com/WebAudio/web-audio-api/issues/1933 and
589 // https://bugzilla.mozilla.org/show_bug.cgi?id=1619486
590 return;
591 }
592 JS::AutoCheckCannotGC nogc;
593 bool isShared;
594 const float* src =
595 JS_GetFloat32ArrayData(float32Arrays[c], &isShared, nogc);
596 MOZ_ASSERT(!isShared); // Was created as unshared
597 PodCopy(output->ChannelFloatsForWrite(c), src, WEBAUDIO_BLOCK_SIZE);
598 }
599 }
600
601 produceSilenceWithError.release(); // have output and no error
602 }
603
AudioWorkletNode(AudioContext * aAudioContext,const nsAString & aName,const AudioWorkletNodeOptions & aOptions)604 AudioWorkletNode::AudioWorkletNode(AudioContext* aAudioContext,
605 const nsAString& aName,
606 const AudioWorkletNodeOptions& aOptions)
607 : AudioNode(aAudioContext, 2, ChannelCountMode::Max,
608 ChannelInterpretation::Speakers),
609 mNodeName(aName),
610 mInputCount(aOptions.mNumberOfInputs),
611 mOutputCount(aOptions.mNumberOfOutputs) {}
612
InitializeParameters(nsTArray<NamedAudioParamTimeline> * aParamTimelines,ErrorResult & aRv)613 void AudioWorkletNode::InitializeParameters(
614 nsTArray<NamedAudioParamTimeline>* aParamTimelines, ErrorResult& aRv) {
615 MOZ_ASSERT(!mParameters, "Only initialize the `parameters` attribute once.");
616 MOZ_ASSERT(aParamTimelines);
617
618 AudioContext* context = Context();
619 const AudioParamDescriptorMap* parameterDescriptors =
620 context->GetParamMapForWorkletName(mNodeName);
621 MOZ_ASSERT(parameterDescriptors);
622
623 size_t audioParamIndex = 0;
624 aParamTimelines->SetCapacity(parameterDescriptors->Length());
625
626 for (size_t i = 0; i < parameterDescriptors->Length(); i++) {
627 auto& paramEntry = (*parameterDescriptors)[i];
628 CreateAudioParam(audioParamIndex++, paramEntry.mName,
629 paramEntry.mDefaultValue, paramEntry.mMinValue,
630 paramEntry.mMaxValue);
631 aParamTimelines->AppendElement(paramEntry);
632 }
633 }
634
SendParameterData(const Optional<Record<nsString,double>> & aParameterData)635 void AudioWorkletNode::SendParameterData(
636 const Optional<Record<nsString, double>>& aParameterData) {
637 MOZ_ASSERT(mTrack, "This method only works if the track has been created.");
638 nsAutoString name;
639 if (aParameterData.WasPassed()) {
640 const auto& paramData = aParameterData.Value();
641 for (const auto& paramDataEntry : paramData.Entries()) {
642 for (auto& audioParam : mParams) {
643 audioParam->GetName(name);
644 if (paramDataEntry.mKey.Equals(name)) {
645 audioParam->SetInitialValue(paramDataEntry.mValue);
646 }
647 }
648 }
649 }
650 }
651
652 /* static */
Constructor(const GlobalObject & aGlobal,AudioContext & aAudioContext,const nsAString & aName,const AudioWorkletNodeOptions & aOptions,ErrorResult & aRv)653 already_AddRefed<AudioWorkletNode> AudioWorkletNode::Constructor(
654 const GlobalObject& aGlobal, AudioContext& aAudioContext,
655 const nsAString& aName, const AudioWorkletNodeOptions& aOptions,
656 ErrorResult& aRv) {
657 /**
658 * 1. If nodeName does not exist as a key in the BaseAudioContext’s node
659 * name to parameter descriptor map, throw a InvalidStateError exception
660 * and abort these steps.
661 */
662 const AudioParamDescriptorMap* parameterDescriptors =
663 aAudioContext.GetParamMapForWorkletName(aName);
664 if (!parameterDescriptors) {
665 // Not using nsPrintfCString in case aName has embedded nulls.
666 aRv.ThrowInvalidStateError("Unknown AudioWorklet name '"_ns +
667 NS_ConvertUTF16toUTF8(aName) + "'"_ns);
668 return nullptr;
669 }
670
671 // See https://github.com/WebAudio/web-audio-api/issues/2074 for ordering.
672 RefPtr<AudioWorkletNode> audioWorkletNode =
673 new AudioWorkletNode(&aAudioContext, aName, aOptions);
674 audioWorkletNode->Initialize(aOptions, aRv);
675 if (NS_WARN_IF(aRv.Failed())) {
676 return nullptr;
677 }
678
679 /**
680 * 3. Configure input, output and output channels of node with options.
681 */
682 if (aOptions.mNumberOfInputs == 0 && aOptions.mNumberOfOutputs == 0) {
683 aRv.ThrowNotSupportedError(
684 "Must have nonzero numbers of inputs or outputs");
685 return nullptr;
686 }
687
688 if (aOptions.mOutputChannelCount.WasPassed()) {
689 /**
690 * 1. If any value in outputChannelCount is zero or greater than the
691 * implementation’s maximum number of channels, throw a
692 * NotSupportedError and abort the remaining steps.
693 */
694 for (uint32_t channelCount : aOptions.mOutputChannelCount.Value()) {
695 if (channelCount == 0 || channelCount > WebAudioUtils::MaxChannelCount) {
696 aRv.ThrowNotSupportedError(
697 nsPrintfCString("Channel count (%u) must be in the range [1, max "
698 "supported channel count]",
699 channelCount));
700 return nullptr;
701 }
702 }
703 /**
704 * 2. If the length of outputChannelCount does not equal numberOfOutputs,
705 * throw an IndexSizeError and abort the remaining steps.
706 */
707 if (aOptions.mOutputChannelCount.Value().Length() !=
708 aOptions.mNumberOfOutputs) {
709 aRv.ThrowIndexSizeError(
710 nsPrintfCString("Length of outputChannelCount (%zu) does not match "
711 "numberOfOutputs (%u)",
712 aOptions.mOutputChannelCount.Value().Length(),
713 aOptions.mNumberOfOutputs));
714 return nullptr;
715 }
716 }
717 // MTG does not support more than UINT16_MAX inputs or outputs.
718 if (aOptions.mNumberOfInputs > UINT16_MAX) {
719 aRv.ThrowRangeError<MSG_VALUE_OUT_OF_RANGE>("numberOfInputs");
720 return nullptr;
721 }
722 if (aOptions.mNumberOfOutputs > UINT16_MAX) {
723 aRv.ThrowRangeError<MSG_VALUE_OUT_OF_RANGE>("numberOfOutputs");
724 return nullptr;
725 }
726
727 /**
728 * 4. Let messageChannel be a new MessageChannel.
729 */
730 RefPtr<MessageChannel> messageChannel =
731 MessageChannel::Constructor(aGlobal, aRv);
732 if (NS_WARN_IF(aRv.Failed())) {
733 return nullptr;
734 }
735 /* 5. Let nodePort be the value of messageChannel’s port1 attribute.
736 * 6. Let processorPortOnThisSide be the value of messageChannel’s port2
737 * attribute.
738 * 7. Let serializedProcessorPort be the result of
739 * StructuredSerializeWithTransfer(processorPortOnThisSide,
740 * « processorPortOnThisSide »).
741 */
742 UniqueMessagePortId processorPortId;
743 messageChannel->Port2()->CloneAndDisentangle(processorPortId);
744 /**
745 * 8. Convert options dictionary to optionsObject.
746 */
747 JSContext* cx = aGlobal.Context();
748 JS::Rooted<JS::Value> optionsVal(cx);
749 if (NS_WARN_IF(!ToJSValue(cx, aOptions, &optionsVal))) {
750 aRv.NoteJSContextException(cx);
751 return nullptr;
752 }
753
754 /**
755 * 9. Let serializedOptions be the result of
756 * StructuredSerialize(optionsObject).
757 */
758
759 // This context and the worklet are part of the same agent cluster and they
760 // can share memory.
761 JS::CloneDataPolicy cloneDataPolicy;
762 cloneDataPolicy.allowIntraClusterClonableSharedObjects();
763 cloneDataPolicy.allowSharedMemoryObjects();
764
765 // StructuredCloneHolder does not have a move constructor. Instead allocate
766 // memory so that the pointer can be passed to the rendering thread.
767 UniquePtr<StructuredCloneHolder> serializedOptions =
768 MakeUnique<StructuredCloneHolder>(
769 StructuredCloneHolder::CloningSupported,
770 StructuredCloneHolder::TransferringNotSupported,
771 JS::StructuredCloneScope::SameProcess);
772 serializedOptions->Write(cx, optionsVal, JS::UndefinedHandleValue,
773 cloneDataPolicy, aRv);
774 if (NS_WARN_IF(aRv.Failed())) {
775 return nullptr;
776 }
777 /**
778 * 10. Set node’s port to nodePort.
779 */
780 audioWorkletNode->mPort = messageChannel->Port1();
781
782 /**
783 * 11. Let parameterDescriptors be the result of retrieval of nodeName from
784 * node name to parameter descriptor map.
785 */
786 nsTArray<NamedAudioParamTimeline> paramTimelines;
787 audioWorkletNode->InitializeParameters(¶mTimelines, aRv);
788 if (NS_WARN_IF(aRv.Failed())) {
789 return nullptr;
790 }
791
792 auto engine = new WorkletNodeEngine(
793 audioWorkletNode, aAudioContext.Destination(), std::move(paramTimelines),
794 aOptions.mOutputChannelCount);
795 audioWorkletNode->mTrack = AudioNodeTrack::Create(
796 &aAudioContext, engine, AudioNodeTrack::NO_TRACK_FLAGS,
797 aAudioContext.Graph());
798
799 audioWorkletNode->SendParameterData(aOptions.mParameterData);
800
801 /**
802 * 12. Queue a control message to invoke the constructor of the
803 * corresponding AudioWorkletProcessor with the processor construction
804 * data that consists of: nodeName, node, serializedOptions, and
805 * serializedProcessorPort.
806 */
807 Worklet* worklet = aAudioContext.GetAudioWorklet(aRv);
808 MOZ_ASSERT(worklet, "Worklet already existed and so getter shouldn't fail.");
809 auto workletImpl = static_cast<AudioWorkletImpl*>(worklet->Impl());
810 audioWorkletNode->mTrack->SendRunnable(NS_NewRunnableFunction(
811 "WorkletNodeEngine::ConstructProcessor",
812 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
813 // See bug 1535398.
814 //
815 // Note that clang and gcc have mutually incompatible rules about whether
816 // attributes should come before or after the `mutable` keyword here, so
817 // use a compatibility hack until we can switch to the standardized
818 // [[attr]] syntax (bug 1627007).
819 #ifdef __clang__
820 # define AND_MUTABLE(macro) macro mutable
821 #else
822 # define AND_MUTABLE(macro) mutable macro
823 #endif
824 [track = audioWorkletNode->mTrack,
825 workletImpl = RefPtr<AudioWorkletImpl>(workletImpl),
826 name = nsString(aName), options = std::move(serializedOptions),
827 portId = std::move(processorPortId)]()
828 AND_MUTABLE(MOZ_CAN_RUN_SCRIPT_BOUNDARY) {
829 auto engine = static_cast<WorkletNodeEngine*>(track->Engine());
830 engine->ConstructProcessor(
831 workletImpl, name, WrapNotNull(options.get()), portId, track);
832 }));
833 #undef AND_MUTABLE
834
835 // [[active source]] is initially true and so at least the first process()
836 // call will not be skipped when there are no active inputs.
837 audioWorkletNode->MarkActive();
838
839 return audioWorkletNode.forget();
840 }
841
GetParameters(ErrorResult & aRv)842 AudioParamMap* AudioWorkletNode::GetParameters(ErrorResult& aRv) {
843 if (!mParameters) {
844 RefPtr<AudioParamMap> parameters = new AudioParamMap(this);
845 nsAutoString name;
846 for (const auto& audioParam : mParams) {
847 audioParam->GetName(name);
848 AudioParamMap_Binding::MaplikeHelpers::Set(parameters, name, *audioParam,
849 aRv);
850 if (NS_WARN_IF(aRv.Failed())) {
851 return nullptr;
852 }
853 }
854 mParameters = std::move(parameters);
855 }
856 return mParameters.get();
857 }
858
DispatchProcessorErrorEvent(const ProcessorErrorDetails & aDetails)859 void AudioWorkletNode::DispatchProcessorErrorEvent(
860 const ProcessorErrorDetails& aDetails) {
861 if (HasListenersFor(nsGkAtoms::onprocessorerror)) {
862 RootedDictionary<ErrorEventInit> init(RootingCx());
863 init.mMessage = aDetails.mMessage;
864 init.mFilename = aDetails.mFilename;
865 init.mLineno = aDetails.mLineno;
866 init.mColno = aDetails.mColno;
867 RefPtr<ErrorEvent> errorEvent =
868 ErrorEvent::Constructor(this, u"processorerror"_ns, init);
869 MOZ_ASSERT(errorEvent);
870 DispatchTrustedEvent(errorEvent);
871 }
872 }
873
WrapObject(JSContext * aCx,JS::Handle<JSObject * > aGivenProto)874 JSObject* AudioWorkletNode::WrapObject(JSContext* aCx,
875 JS::Handle<JSObject*> aGivenProto) {
876 return AudioWorkletNode_Binding::Wrap(aCx, this, aGivenProto);
877 }
878
SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const879 size_t AudioWorkletNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
880 size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf);
881 return amount;
882 }
883
SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const884 size_t AudioWorkletNode::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
885 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
886 }
887
888 } // namespace mozilla::dom
889