1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaBufferDecoder.h"
8
9 #include <speex/speex_resampler.h>
10
11 #include "AudioBuffer.h"
12 #include "AudioContext.h"
13 #include "AudioNodeEngine.h"
14 #include "BufferMediaResource.h"
15 #include "DecoderTraits.h"
16 #include "MediaContainerType.h"
17 #include "MediaDataDecoderProxy.h"
18 #include "MediaDataDemuxer.h"
19 #include "MediaQueue.h"
20 #include "PDMFactory.h"
21 #include "VideoUtils.h"
22 #include "WebAudioUtils.h"
23 #include "js/MemoryFunctions.h"
24 #include "mozilla/AbstractThread.h"
25 #include "mozilla/Logging.h"
26 #include "mozilla/StaticPrefs_media.h"
27 #include "mozilla/Telemetry.h"
28 #include "mozilla/dom/AudioContextBinding.h"
29 #include "mozilla/dom/BaseAudioContextBinding.h"
30 #include "mozilla/dom/DOMException.h"
31 #include "mozilla/dom/Promise.h"
32 #include "mozilla/dom/ScriptSettings.h"
33 #include "nsComponentManagerUtils.h"
34 #include "nsContentUtils.h"
35 #include "nsIScriptError.h"
36 #include "nsIScriptObjectPrincipal.h"
37 #include "nsMimeTypes.h"
38 #include "nsPrintfCString.h"
39 #include "nsXPCOMCIDInternal.h"
40
41 namespace mozilla {
42
43 extern LazyLogModule gMediaDecoderLog;
44
45 #define LOG(x, ...) \
46 MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (x, ##__VA_ARGS__))
47
48 using namespace dom;
49
50 class ReportResultTask final : public Runnable {
51 public:
ReportResultTask(WebAudioDecodeJob & aDecodeJob,WebAudioDecodeJob::ResultFn aFunction,WebAudioDecodeJob::ErrorCode aErrorCode)52 ReportResultTask(WebAudioDecodeJob& aDecodeJob,
53 WebAudioDecodeJob::ResultFn aFunction,
54 WebAudioDecodeJob::ErrorCode aErrorCode)
55 : Runnable("ReportResultTask"),
56 mDecodeJob(aDecodeJob),
57 mFunction(aFunction),
58 mErrorCode(aErrorCode) {
59 MOZ_ASSERT(aFunction);
60 }
61
Run()62 NS_IMETHOD Run() override {
63 MOZ_ASSERT(NS_IsMainThread());
64
65 (mDecodeJob.*mFunction)(mErrorCode);
66
67 return NS_OK;
68 }
69
70 private:
71 // Note that the mDecodeJob member will probably die when mFunction is run.
72 // Therefore, it is not safe to do anything fancy with it in this class.
73 // Really, this class is only used because nsRunnableMethod doesn't support
74 // methods accepting arguments.
75 WebAudioDecodeJob& mDecodeJob;
76 WebAudioDecodeJob::ResultFn mFunction;
77 WebAudioDecodeJob::ErrorCode mErrorCode;
78 };
79
80 enum class PhaseEnum : int { Decode, AllocateBuffer, Done };
81
82 class MediaDecodeTask final : public Runnable {
83 public:
MediaDecodeTask(const MediaContainerType & aContainerType,uint8_t * aBuffer,uint32_t aLength,WebAudioDecodeJob & aDecodeJob)84 MediaDecodeTask(const MediaContainerType& aContainerType, uint8_t* aBuffer,
85 uint32_t aLength, WebAudioDecodeJob& aDecodeJob)
86 : Runnable("MediaDecodeTask"),
87 mContainerType(aContainerType),
88 mBuffer(aBuffer),
89 mLength(aLength),
90 mBatchSize(StaticPrefs::media_rdd_webaudio_batch_size()),
91 mDecodeJob(aDecodeJob),
92 mPhase(PhaseEnum::Decode) {
93 MOZ_ASSERT(aBuffer);
94 MOZ_ASSERT(NS_IsMainThread());
95 }
96
97 // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. See
98 // bug 1535398.
99 MOZ_CAN_RUN_SCRIPT_BOUNDARY
100 NS_IMETHOD Run() override;
101 bool Init();
PSupervisorTaskQueue()102 TaskQueue* PSupervisorTaskQueue() { return mPSupervisorTaskQueue; }
OnPSupervisorTaskQueue() const103 bool OnPSupervisorTaskQueue() const {
104 return mPSupervisorTaskQueue->IsCurrentThreadIn();
105 }
106
107 private:
108 MOZ_CAN_RUN_SCRIPT
ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode)109 void ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode) {
110 if (NS_IsMainThread()) {
111 Cleanup();
112 mDecodeJob.OnFailure(aErrorCode);
113 } else {
114 // Take extra care to cleanup on the main thread
115 mMainThread->Dispatch(NewRunnableMethod("MediaDecodeTask::Cleanup", this,
116 &MediaDecodeTask::Cleanup));
117
118 nsCOMPtr<nsIRunnable> event = new ReportResultTask(
119 mDecodeJob, &WebAudioDecodeJob::OnFailure, aErrorCode);
120 mMainThread->Dispatch(event.forget());
121 }
122 }
123
124 void Decode();
125
126 void OnCreateDecoderCompleted(RefPtr<MediaDataDecoder> aDecoder);
127 MOZ_CAN_RUN_SCRIPT void OnCreateDecoderFailed(const MediaResult& aError);
128
129 MOZ_CAN_RUN_SCRIPT void OnInitDemuxerCompleted();
130 MOZ_CAN_RUN_SCRIPT void OnInitDemuxerFailed(const MediaResult& aError);
131
132 void InitDecoder();
133 void OnInitDecoderCompleted();
134 MOZ_CAN_RUN_SCRIPT void OnInitDecoderFailed();
135
136 void DoDemux();
137 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
138 MOZ_CAN_RUN_SCRIPT void OnAudioDemuxFailed(const MediaResult& aError);
139
140 void DoDecode();
141 void OnAudioDecodeCompleted(MediaDataDecoder::DecodedData&& aResults);
142 MOZ_CAN_RUN_SCRIPT void OnAudioDecodeFailed(const MediaResult& aError);
143
144 void DoDrain();
145 MOZ_CAN_RUN_SCRIPT void OnAudioDrainCompleted(
146 MediaDataDecoder::DecodedData&& aResults);
147 MOZ_CAN_RUN_SCRIPT void OnAudioDrainFailed(const MediaResult& aError);
148
149 void ShutdownDecoder();
150
151 MOZ_CAN_RUN_SCRIPT void FinishDecode();
152 MOZ_CAN_RUN_SCRIPT void AllocateBuffer();
153 MOZ_CAN_RUN_SCRIPT void CallbackTheResult();
154
Cleanup()155 void Cleanup() {
156 MOZ_ASSERT(NS_IsMainThread());
157 JS_free(nullptr, mBuffer);
158 if (mTrackDemuxer) {
159 mTrackDemuxer->BreakCycles();
160 }
161 mTrackDemuxer = nullptr;
162 mDemuxer = nullptr;
163 mPSupervisorTaskQueue = nullptr;
164 mPDecoderTaskQueue = nullptr;
165 }
166
167 private:
168 MediaContainerType mContainerType;
169 uint8_t* mBuffer;
170 const uint32_t mLength;
171 const uint32_t mBatchSize;
172 WebAudioDecodeJob& mDecodeJob;
173 PhaseEnum mPhase;
174 RefPtr<TaskQueue> mPSupervisorTaskQueue;
175 RefPtr<TaskQueue> mPDecoderTaskQueue;
176 RefPtr<MediaDataDemuxer> mDemuxer;
177 RefPtr<MediaTrackDemuxer> mTrackDemuxer;
178 RefPtr<MediaDataDecoder> mDecoder;
179 nsTArray<RefPtr<MediaRawData>> mRawSamples;
180 MediaInfo mMediaInfo;
181 MediaQueue<AudioData> mAudioQueue;
182 RefPtr<AbstractThread> mMainThread;
183 };
184
185 NS_IMETHODIMP
Run()186 MediaDecodeTask::Run() {
187 switch (mPhase) {
188 case PhaseEnum::Decode:
189 Decode();
190 break;
191 case PhaseEnum::AllocateBuffer:
192 AllocateBuffer();
193 break;
194 case PhaseEnum::Done:
195 break;
196 }
197
198 return NS_OK;
199 }
200
Init()201 bool MediaDecodeTask::Init() {
202 MOZ_ASSERT(NS_IsMainThread());
203
204 RefPtr<BufferMediaResource> resource =
205 new BufferMediaResource(static_cast<uint8_t*>(mBuffer), mLength);
206
207 mMainThread = mDecodeJob.mContext->GetOwnerGlobal()->AbstractMainThreadFor(
208 TaskCategory::Other);
209
210 mPSupervisorTaskQueue =
211 new TaskQueue(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
212 "MediaBufferDecoder::mPSupervisorTaskQueue");
213 mPDecoderTaskQueue =
214 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
215 "MediaBufferDecoder::mPDecoderTaskQueue");
216
217 // If you change this list to add support for new decoders, please consider
218 // updating HTMLMediaElement::CreateDecoder as well.
219 mDemuxer = DecoderTraits::CreateDemuxer(mContainerType, resource);
220 if (!mDemuxer) {
221 return false;
222 }
223
224 return true;
225 }
226
227 class AutoResampler final {
228 public:
AutoResampler()229 AutoResampler() : mResampler(nullptr) {}
~AutoResampler()230 ~AutoResampler() {
231 if (mResampler) {
232 speex_resampler_destroy(mResampler);
233 }
234 }
operator SpeexResamplerState*() const235 operator SpeexResamplerState*() const {
236 MOZ_ASSERT(mResampler);
237 return mResampler;
238 }
operator =(SpeexResamplerState * aResampler)239 void operator=(SpeexResamplerState* aResampler) { mResampler = aResampler; }
240
241 private:
242 SpeexResamplerState* mResampler;
243 };
244
Decode()245 void MediaDecodeTask::Decode() {
246 MOZ_ASSERT(OnPSupervisorTaskQueue());
247
248 mDemuxer->Init()->Then(PSupervisorTaskQueue(), __func__, this,
249 &MediaDecodeTask::OnInitDemuxerCompleted,
250 &MediaDecodeTask::OnInitDemuxerFailed);
251 }
252
OnInitDemuxerCompleted()253 void MediaDecodeTask::OnInitDemuxerCompleted() {
254 MOZ_ASSERT(OnPSupervisorTaskQueue());
255
256 if (!!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack)) {
257 mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
258 if (!mTrackDemuxer) {
259 LOG("MediaDecodeTask: Could not get a track demuxer.");
260 ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
261 return;
262 }
263
264 RefPtr<PDMFactory> platform = new PDMFactory();
265 UniquePtr<TrackInfo> audioInfo = mTrackDemuxer->GetInfo();
266 // We actively ignore audio tracks that we know we can't play.
267 if (audioInfo && audioInfo->IsValid() &&
268 platform->SupportsMimeType(audioInfo->mMimeType)) {
269 mMediaInfo.mAudio = *audioInfo->GetAsAudioInfo();
270 }
271 }
272
273 RefPtr<PDMFactory> pdm = new PDMFactory();
274 pdm->CreateDecoder(
275 {*mMediaInfo.mAudio.GetAsAudioInfo(), TrackInfo::kAudioTrack})
276 ->Then(PSupervisorTaskQueue(), __func__, this,
277 &MediaDecodeTask::OnCreateDecoderCompleted,
278 &MediaDecodeTask::OnCreateDecoderFailed);
279 }
280
OnCreateDecoderCompleted(RefPtr<MediaDataDecoder> aDecoder)281 void MediaDecodeTask::OnCreateDecoderCompleted(
282 RefPtr<MediaDataDecoder> aDecoder) {
283 MOZ_ASSERT(OnPSupervisorTaskQueue());
284
285 mDecoder = new MediaDataDecoderProxy(aDecoder.forget(),
286 do_AddRef(mPDecoderTaskQueue.get()));
287 InitDecoder();
288 }
289
OnCreateDecoderFailed(const MediaResult & aError)290 void MediaDecodeTask::OnCreateDecoderFailed(const MediaResult& aError) {
291 MOZ_ASSERT(OnPSupervisorTaskQueue());
292
293 LOG("MediaDecodeTask: Could not create a decoder.");
294 ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
295 }
296
OnInitDemuxerFailed(const MediaResult & aError)297 void MediaDecodeTask::OnInitDemuxerFailed(const MediaResult& aError) {
298 MOZ_ASSERT(OnPSupervisorTaskQueue());
299
300 LOG("MediaDecodeTask: Could not initialize the demuxer.");
301 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
302 }
303
InitDecoder()304 void MediaDecodeTask::InitDecoder() {
305 MOZ_ASSERT(OnPSupervisorTaskQueue());
306
307 mDecoder->Init()->Then(PSupervisorTaskQueue(), __func__, this,
308 &MediaDecodeTask::OnInitDecoderCompleted,
309 &MediaDecodeTask::OnInitDecoderFailed);
310 }
311
OnInitDecoderCompleted()312 void MediaDecodeTask::OnInitDecoderCompleted() {
313 MOZ_ASSERT(OnPSupervisorTaskQueue());
314
315 DoDemux();
316 }
317
OnInitDecoderFailed()318 void MediaDecodeTask::OnInitDecoderFailed() {
319 MOZ_ASSERT(OnPSupervisorTaskQueue());
320
321 ShutdownDecoder();
322 LOG("MediaDecodeTask: Could not initialize the decoder");
323 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
324 }
325
DoDemux()326 void MediaDecodeTask::DoDemux() {
327 MOZ_ASSERT(OnPSupervisorTaskQueue());
328
329 mTrackDemuxer->GetSamples(mBatchSize)
330 ->Then(PSupervisorTaskQueue(), __func__, this,
331 &MediaDecodeTask::OnAudioDemuxCompleted,
332 &MediaDecodeTask::OnAudioDemuxFailed);
333 }
334
OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)335 void MediaDecodeTask::OnAudioDemuxCompleted(
336 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
337 MOZ_ASSERT(OnPSupervisorTaskQueue());
338
339 mRawSamples.AppendElements(aSamples->GetSamples());
340
341 DoDemux();
342 }
343
OnAudioDemuxFailed(const MediaResult & aError)344 void MediaDecodeTask::OnAudioDemuxFailed(const MediaResult& aError) {
345 MOZ_ASSERT(OnPSupervisorTaskQueue());
346
347 if (aError.Code() == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
348 DoDecode();
349 } else {
350 ShutdownDecoder();
351 LOG("MediaDecodeTask: Audio demux failed");
352 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
353 }
354 }
355
DoDecode()356 void MediaDecodeTask::DoDecode() {
357 MOZ_ASSERT(OnPSupervisorTaskQueue());
358
359 if (mRawSamples.IsEmpty()) {
360 DoDrain();
361 return;
362 }
363
364 if (mBatchSize > 1 && mDecoder->CanDecodeBatch()) {
365 nsTArray<RefPtr<MediaRawData>> rawSampleBatch;
366 const int batchSize = std::min((unsigned long)mBatchSize,
367 (unsigned long)mRawSamples.Length());
368 for (int i = 0; i < batchSize; ++i) {
369 rawSampleBatch.AppendElement(std::move(mRawSamples[i]));
370 }
371
372 mDecoder->DecodeBatch(std::move(rawSampleBatch))
373 ->Then(PSupervisorTaskQueue(), __func__, this,
374 &MediaDecodeTask::OnAudioDecodeCompleted,
375 &MediaDecodeTask::OnAudioDecodeFailed);
376
377 mRawSamples.RemoveElementsAt(0, batchSize);
378 } else {
379 RefPtr<MediaRawData> sample = std::move(mRawSamples[0]);
380
381 mDecoder->Decode(sample)->Then(PSupervisorTaskQueue(), __func__, this,
382 &MediaDecodeTask::OnAudioDecodeCompleted,
383 &MediaDecodeTask::OnAudioDecodeFailed);
384
385 mRawSamples.RemoveElementAt(0);
386 }
387 }
388
OnAudioDecodeCompleted(MediaDataDecoder::DecodedData && aResults)389 void MediaDecodeTask::OnAudioDecodeCompleted(
390 MediaDataDecoder::DecodedData&& aResults) {
391 MOZ_ASSERT(OnPSupervisorTaskQueue());
392
393 for (auto&& sample : aResults) {
394 MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
395 RefPtr<AudioData> audioData = sample->As<AudioData>();
396
397 mMediaInfo.mAudio.mRate = audioData->mRate;
398 mMediaInfo.mAudio.mChannels = audioData->mChannels;
399
400 mAudioQueue.Push(audioData.forget());
401 }
402
403 DoDecode();
404 }
405
OnAudioDecodeFailed(const MediaResult & aError)406 void MediaDecodeTask::OnAudioDecodeFailed(const MediaResult& aError) {
407 MOZ_ASSERT(OnPSupervisorTaskQueue());
408
409 ShutdownDecoder();
410 LOG("MediaDecodeTask: decode audio failed.");
411 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
412 }
413
DoDrain()414 void MediaDecodeTask::DoDrain() {
415 MOZ_ASSERT(OnPSupervisorTaskQueue());
416
417 mDecoder->Drain()->Then(PSupervisorTaskQueue(), __func__, this,
418 &MediaDecodeTask::OnAudioDrainCompleted,
419 &MediaDecodeTask::OnAudioDrainFailed);
420 }
421
OnAudioDrainCompleted(MediaDataDecoder::DecodedData && aResults)422 void MediaDecodeTask::OnAudioDrainCompleted(
423 MediaDataDecoder::DecodedData&& aResults) {
424 MOZ_ASSERT(OnPSupervisorTaskQueue());
425
426 if (aResults.IsEmpty()) {
427 FinishDecode();
428 return;
429 }
430
431 for (auto&& sample : aResults) {
432 MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
433 RefPtr<AudioData> audioData = sample->As<AudioData>();
434
435 mAudioQueue.Push(audioData.forget());
436 }
437 DoDrain();
438 }
439
OnAudioDrainFailed(const MediaResult & aError)440 void MediaDecodeTask::OnAudioDrainFailed(const MediaResult& aError) {
441 MOZ_ASSERT(OnPSupervisorTaskQueue());
442
443 ShutdownDecoder();
444 LOG("MediaDecodeTask: Drain audio failed");
445 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
446 }
447
ShutdownDecoder()448 void MediaDecodeTask::ShutdownDecoder() {
449 MOZ_ASSERT(OnPSupervisorTaskQueue());
450
451 if (!mDecoder) {
452 return;
453 }
454
455 RefPtr<MediaDecodeTask> self = this;
456 mDecoder->Shutdown();
457 mDecoder = nullptr;
458 }
459
FinishDecode()460 void MediaDecodeTask::FinishDecode() {
461 MOZ_ASSERT(OnPSupervisorTaskQueue());
462
463 ShutdownDecoder();
464
465 uint32_t frameCount = mAudioQueue.AudioFramesCount();
466 uint32_t channelCount = mMediaInfo.mAudio.mChannels;
467 uint32_t sampleRate = mMediaInfo.mAudio.mRate;
468
469 if (!frameCount || !channelCount || !sampleRate) {
470 LOG("MediaDecodeTask: invalid content frame count, channel count or "
471 "sample-rate");
472 ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
473 return;
474 }
475
476 const uint32_t destSampleRate = mDecodeJob.mContext->SampleRate();
477 AutoResampler resampler;
478
479 uint32_t resampledFrames = frameCount;
480 if (sampleRate != destSampleRate) {
481 resampledFrames = static_cast<uint32_t>(
482 static_cast<uint64_t>(destSampleRate) *
483 static_cast<uint64_t>(frameCount) / static_cast<uint64_t>(sampleRate));
484
485 resampler = speex_resampler_init(channelCount, sampleRate, destSampleRate,
486 SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr);
487 speex_resampler_skip_zeros(resampler);
488 resampledFrames += speex_resampler_get_output_latency(resampler);
489 }
490
491 // Allocate contiguous channel buffers. Note that if we end up resampling,
492 // we may write fewer bytes than mResampledFrames to the output buffer, in
493 // which case writeIndex will tell us how many valid samples we have.
494 mDecodeJob.mBuffer.mChannelData.SetLength(channelCount);
495 #if AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_FLOAT32
496 // This buffer has separate channel arrays that could be transferred to
497 // JS::NewArrayBufferWithContents(), but AudioBuffer::RestoreJSChannelData()
498 // does not yet take advantage of this.
499 RefPtr<ThreadSharedFloatArrayBufferList> buffer =
500 ThreadSharedFloatArrayBufferList::Create(channelCount, resampledFrames,
501 fallible);
502 if (!buffer) {
503 LOG("MediaDecodeTask: Could not create final buffer (f32)");
504 ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
505 return;
506 }
507 for (uint32_t i = 0; i < channelCount; ++i) {
508 mDecodeJob.mBuffer.mChannelData[i] = buffer->GetData(i);
509 }
510 #else
511 CheckedInt<size_t> bufferSize(sizeof(AudioDataValue));
512 bufferSize *= resampledFrames;
513 bufferSize *= channelCount;
514 RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize);
515 if (!buffer) {
516 LOG("MediaDecodeTask: Could not create final buffer (i16)");
517 ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
518 return;
519 }
520 auto data = static_cast<AudioDataValue*>(floatBuffer->Data());
521 for (uint32_t i = 0; i < channelCount; ++i) {
522 mDecodeJob.mBuffer.mChannelData[i] = data;
523 data += resampledFrames;
524 }
525 #endif
526 mDecodeJob.mBuffer.mBuffer = std::move(buffer);
527 mDecodeJob.mBuffer.mVolume = 1.0f;
528 mDecodeJob.mBuffer.mBufferFormat = AUDIO_OUTPUT_FORMAT;
529
530 uint32_t writeIndex = 0;
531 RefPtr<AudioData> audioData;
532 while ((audioData = mAudioQueue.PopFront())) {
533 if (!audioData->Frames()) {
534 // The packet contains no audio frames, skip it.
535 continue;
536 }
537 audioData->EnsureAudioBuffer(); // could lead to a copy :(
538 const AudioDataValue* bufferData =
539 static_cast<AudioDataValue*>(audioData->mAudioBuffer->Data());
540
541 if (sampleRate != destSampleRate) {
542 const uint32_t maxOutSamples = resampledFrames - writeIndex;
543
544 for (uint32_t i = 0; i < audioData->mChannels; ++i) {
545 uint32_t inSamples = audioData->Frames();
546 uint32_t outSamples = maxOutSamples;
547 AudioDataValue* outData =
548 mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
549 writeIndex;
550
551 WebAudioUtils::SpeexResamplerProcess(
552 resampler, i, &bufferData[i * audioData->Frames()], &inSamples,
553 outData, &outSamples);
554
555 if (i == audioData->mChannels - 1) {
556 writeIndex += outSamples;
557 MOZ_ASSERT(writeIndex <= resampledFrames);
558 MOZ_ASSERT(inSamples == audioData->Frames());
559 }
560 }
561 } else {
562 for (uint32_t i = 0; i < audioData->mChannels; ++i) {
563 AudioDataValue* outData =
564 mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
565 writeIndex;
566 PodCopy(outData, &bufferData[i * audioData->Frames()],
567 audioData->Frames());
568
569 if (i == audioData->mChannels - 1) {
570 writeIndex += audioData->Frames();
571 }
572 }
573 }
574 }
575
576 if (sampleRate != destSampleRate) {
577 uint32_t inputLatency = speex_resampler_get_input_latency(resampler);
578 const uint32_t maxOutSamples = resampledFrames - writeIndex;
579 for (uint32_t i = 0; i < channelCount; ++i) {
580 uint32_t inSamples = inputLatency;
581 uint32_t outSamples = maxOutSamples;
582 AudioDataValue* outData =
583 mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
584 writeIndex;
585
586 WebAudioUtils::SpeexResamplerProcess(resampler, i,
587 (AudioDataValue*)nullptr, &inSamples,
588 outData, &outSamples);
589
590 if (i == channelCount - 1) {
591 writeIndex += outSamples;
592 MOZ_ASSERT(writeIndex <= resampledFrames);
593 MOZ_ASSERT(inSamples == inputLatency);
594 }
595 }
596 }
597
598 mDecodeJob.mBuffer.mDuration = writeIndex;
599 mPhase = PhaseEnum::AllocateBuffer;
600 mMainThread->Dispatch(do_AddRef(this));
601 }
602
AllocateBuffer()603 void MediaDecodeTask::AllocateBuffer() {
604 MOZ_ASSERT(NS_IsMainThread());
605
606 if (!mDecodeJob.AllocateBuffer()) {
607 LOG("MediaDecodeTask: Could not allocate final buffer");
608 ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
609 return;
610 }
611
612 mPhase = PhaseEnum::Done;
613 CallbackTheResult();
614 }
615
CallbackTheResult()616 void MediaDecodeTask::CallbackTheResult() {
617 MOZ_ASSERT(NS_IsMainThread());
618
619 Cleanup();
620
621 // Now, we're ready to call the script back with the resulting buffer
622 mDecodeJob.OnSuccess(WebAudioDecodeJob::NoError);
623 }
624
AllocateBuffer()625 bool WebAudioDecodeJob::AllocateBuffer() {
626 MOZ_ASSERT(!mOutput);
627 MOZ_ASSERT(NS_IsMainThread());
628
629 // Now create the AudioBuffer
630 mOutput = AudioBuffer::Create(mContext->GetOwner(), mContext->SampleRate(),
631 std::move(mBuffer));
632 return mOutput != nullptr;
633 }
634
AsyncDecodeWebAudio(const char * aContentType,uint8_t * aBuffer,uint32_t aLength,WebAudioDecodeJob & aDecodeJob)635 void AsyncDecodeWebAudio(const char* aContentType, uint8_t* aBuffer,
636 uint32_t aLength, WebAudioDecodeJob& aDecodeJob) {
637 Maybe<MediaContainerType> containerType =
638 MakeMediaContainerType(aContentType);
639 // Do not attempt to decode the media if we were not successful at sniffing
640 // the container type.
641 if (!*aContentType || strcmp(aContentType, APPLICATION_OCTET_STREAM) == 0 ||
642 !containerType) {
643 nsCOMPtr<nsIRunnable> event =
644 new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
645 WebAudioDecodeJob::UnknownContent);
646 JS_free(nullptr, aBuffer);
647 aDecodeJob.mContext->Dispatch(event.forget());
648 return;
649 }
650
651 RefPtr<MediaDecodeTask> task =
652 new MediaDecodeTask(*containerType, aBuffer, aLength, aDecodeJob);
653 if (!task->Init()) {
654 nsCOMPtr<nsIRunnable> event =
655 new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
656 WebAudioDecodeJob::UnknownError);
657 aDecodeJob.mContext->Dispatch(event.forget());
658 } else {
659 nsresult rv = task->PSupervisorTaskQueue()->Dispatch(task.forget());
660 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
661 Unused << rv;
662 }
663 }
664
WebAudioDecodeJob(AudioContext * aContext,Promise * aPromise,DecodeSuccessCallback * aSuccessCallback,DecodeErrorCallback * aFailureCallback)665 WebAudioDecodeJob::WebAudioDecodeJob(AudioContext* aContext, Promise* aPromise,
666 DecodeSuccessCallback* aSuccessCallback,
667 DecodeErrorCallback* aFailureCallback)
668 : mContext(aContext),
669 mPromise(aPromise),
670 mSuccessCallback(aSuccessCallback),
671 mFailureCallback(aFailureCallback) {
672 MOZ_ASSERT(aContext);
673 MOZ_ASSERT(NS_IsMainThread());
674 MOZ_COUNT_CTOR(WebAudioDecodeJob);
675 }
676
~WebAudioDecodeJob()677 WebAudioDecodeJob::~WebAudioDecodeJob() {
678 MOZ_ASSERT(NS_IsMainThread());
679 MOZ_COUNT_DTOR(WebAudioDecodeJob);
680 }
681
OnSuccess(ErrorCode aErrorCode)682 void WebAudioDecodeJob::OnSuccess(ErrorCode aErrorCode) {
683 MOZ_ASSERT(NS_IsMainThread());
684 MOZ_ASSERT(aErrorCode == NoError);
685
686 RefPtr<AudioBuffer> output(mOutput);
687 if (mSuccessCallback) {
688 RefPtr<DecodeSuccessCallback> callback(mSuccessCallback);
689 // Ignore errors in calling the callback, since there is not much that we
690 // can do about it here.
691 callback->Call(*output);
692 }
693 mPromise->MaybeResolve(output);
694
695 mContext->RemoveFromDecodeQueue(this);
696 }
697
OnFailure(ErrorCode aErrorCode)698 void WebAudioDecodeJob::OnFailure(ErrorCode aErrorCode) {
699 MOZ_ASSERT(NS_IsMainThread());
700
701 const char* errorMessage;
702 switch (aErrorCode) {
703 case UnknownContent:
704 errorMessage =
705 "The buffer passed to decodeAudioData contains an unknown content "
706 "type.";
707 break;
708 case InvalidContent:
709 errorMessage =
710 "The buffer passed to decodeAudioData contains invalid content which "
711 "cannot be decoded successfully.";
712 break;
713 case NoAudio:
714 errorMessage =
715 "The buffer passed to decodeAudioData does not contain any audio.";
716 break;
717 case NoError:
718 MOZ_FALLTHROUGH_ASSERT("Who passed NoError to OnFailure?");
719 // Fall through to get some sort of a sane error message if this actually
720 // happens at runtime.
721 case UnknownError:
722 [[fallthrough]];
723 default:
724 errorMessage =
725 "An unknown error occurred while processing decodeAudioData.";
726 break;
727 }
728
729 // Ignore errors in calling the callback, since there is not much that we can
730 // do about it here.
731 nsAutoCString errorString(errorMessage);
732 if (mFailureCallback) {
733 RefPtr<DOMException> exception = DOMException::Create(
734 NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR, errorString);
735 RefPtr<DecodeErrorCallback> callback(mFailureCallback);
736 callback->Call(*exception);
737 }
738
739 mPromise->MaybeRejectWithEncodingError(errorString);
740
741 mContext->RemoveFromDecodeQueue(this);
742 }
743
SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const744 size_t WebAudioDecodeJob::SizeOfExcludingThis(
745 MallocSizeOf aMallocSizeOf) const {
746 size_t amount = 0;
747 if (mSuccessCallback) {
748 amount += mSuccessCallback->SizeOfIncludingThis(aMallocSizeOf);
749 }
750 if (mFailureCallback) {
751 amount += mFailureCallback->SizeOfIncludingThis(aMallocSizeOf);
752 }
753 if (mOutput) {
754 amount += mOutput->SizeOfIncludingThis(aMallocSizeOf);
755 }
756 amount += mBuffer.SizeOfExcludingThis(aMallocSizeOf, false);
757 return amount;
758 }
759
SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const760 size_t WebAudioDecodeJob::SizeOfIncludingThis(
761 MallocSizeOf aMallocSizeOf) const {
762 return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
763 }
764
765 } // namespace mozilla
766