1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "MediaBufferDecoder.h"
8 
9 #include <speex/speex_resampler.h>
10 
11 #include "AudioBuffer.h"
12 #include "AudioContext.h"
13 #include "AudioNodeEngine.h"
14 #include "BufferMediaResource.h"
15 #include "DecoderTraits.h"
16 #include "MediaContainerType.h"
17 #include "MediaDataDecoderProxy.h"
18 #include "MediaDataDemuxer.h"
19 #include "MediaQueue.h"
20 #include "PDMFactory.h"
21 #include "VideoUtils.h"
22 #include "WebAudioUtils.h"
23 #include "js/MemoryFunctions.h"
24 #include "mozilla/AbstractThread.h"
25 #include "mozilla/Logging.h"
26 #include "mozilla/StaticPrefs_media.h"
27 #include "mozilla/Telemetry.h"
28 #include "mozilla/dom/AudioContextBinding.h"
29 #include "mozilla/dom/BaseAudioContextBinding.h"
30 #include "mozilla/dom/DOMException.h"
31 #include "mozilla/dom/Promise.h"
32 #include "mozilla/dom/ScriptSettings.h"
33 #include "nsComponentManagerUtils.h"
34 #include "nsContentUtils.h"
35 #include "nsIScriptError.h"
36 #include "nsIScriptObjectPrincipal.h"
37 #include "nsMimeTypes.h"
38 #include "nsPrintfCString.h"
39 #include "nsXPCOMCIDInternal.h"
40 
41 namespace mozilla {
42 
43 extern LazyLogModule gMediaDecoderLog;
44 
45 #define LOG(x, ...) \
46   MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (x, ##__VA_ARGS__))
47 
48 using namespace dom;
49 
50 class ReportResultTask final : public Runnable {
51  public:
ReportResultTask(WebAudioDecodeJob & aDecodeJob,WebAudioDecodeJob::ResultFn aFunction,WebAudioDecodeJob::ErrorCode aErrorCode)52   ReportResultTask(WebAudioDecodeJob& aDecodeJob,
53                    WebAudioDecodeJob::ResultFn aFunction,
54                    WebAudioDecodeJob::ErrorCode aErrorCode)
55       : Runnable("ReportResultTask"),
56         mDecodeJob(aDecodeJob),
57         mFunction(aFunction),
58         mErrorCode(aErrorCode) {
59     MOZ_ASSERT(aFunction);
60   }
61 
Run()62   NS_IMETHOD Run() override {
63     MOZ_ASSERT(NS_IsMainThread());
64 
65     (mDecodeJob.*mFunction)(mErrorCode);
66 
67     return NS_OK;
68   }
69 
70  private:
71   // Note that the mDecodeJob member will probably die when mFunction is run.
72   // Therefore, it is not safe to do anything fancy with it in this class.
73   // Really, this class is only used because nsRunnableMethod doesn't support
74   // methods accepting arguments.
75   WebAudioDecodeJob& mDecodeJob;
76   WebAudioDecodeJob::ResultFn mFunction;
77   WebAudioDecodeJob::ErrorCode mErrorCode;
78 };
79 
80 enum class PhaseEnum : int { Decode, AllocateBuffer, Done };
81 
82 class MediaDecodeTask final : public Runnable {
83  public:
MediaDecodeTask(const MediaContainerType & aContainerType,uint8_t * aBuffer,uint32_t aLength,WebAudioDecodeJob & aDecodeJob)84   MediaDecodeTask(const MediaContainerType& aContainerType, uint8_t* aBuffer,
85                   uint32_t aLength, WebAudioDecodeJob& aDecodeJob)
86       : Runnable("MediaDecodeTask"),
87         mContainerType(aContainerType),
88         mBuffer(aBuffer),
89         mLength(aLength),
90         mBatchSize(StaticPrefs::media_rdd_webaudio_batch_size()),
91         mDecodeJob(aDecodeJob),
92         mPhase(PhaseEnum::Decode) {
93     MOZ_ASSERT(aBuffer);
94     MOZ_ASSERT(NS_IsMainThread());
95   }
96 
97   // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.  See
98   // bug 1535398.
99   MOZ_CAN_RUN_SCRIPT_BOUNDARY
100   NS_IMETHOD Run() override;
101   bool Init();
PSupervisorTaskQueue()102   TaskQueue* PSupervisorTaskQueue() { return mPSupervisorTaskQueue; }
OnPSupervisorTaskQueue() const103   bool OnPSupervisorTaskQueue() const {
104     return mPSupervisorTaskQueue->IsCurrentThreadIn();
105   }
106 
107  private:
108   MOZ_CAN_RUN_SCRIPT
ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode)109   void ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode) {
110     if (NS_IsMainThread()) {
111       Cleanup();
112       mDecodeJob.OnFailure(aErrorCode);
113     } else {
114       // Take extra care to cleanup on the main thread
115       mMainThread->Dispatch(NewRunnableMethod("MediaDecodeTask::Cleanup", this,
116                                               &MediaDecodeTask::Cleanup));
117 
118       nsCOMPtr<nsIRunnable> event = new ReportResultTask(
119           mDecodeJob, &WebAudioDecodeJob::OnFailure, aErrorCode);
120       mMainThread->Dispatch(event.forget());
121     }
122   }
123 
124   void Decode();
125 
126   void OnCreateDecoderCompleted(RefPtr<MediaDataDecoder> aDecoder);
127   MOZ_CAN_RUN_SCRIPT void OnCreateDecoderFailed(const MediaResult& aError);
128 
129   MOZ_CAN_RUN_SCRIPT void OnInitDemuxerCompleted();
130   MOZ_CAN_RUN_SCRIPT void OnInitDemuxerFailed(const MediaResult& aError);
131 
132   void InitDecoder();
133   void OnInitDecoderCompleted();
134   MOZ_CAN_RUN_SCRIPT void OnInitDecoderFailed();
135 
136   void DoDemux();
137   void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
138   MOZ_CAN_RUN_SCRIPT void OnAudioDemuxFailed(const MediaResult& aError);
139 
140   void DoDecode();
141   void OnAudioDecodeCompleted(MediaDataDecoder::DecodedData&& aResults);
142   MOZ_CAN_RUN_SCRIPT void OnAudioDecodeFailed(const MediaResult& aError);
143 
144   void DoDrain();
145   MOZ_CAN_RUN_SCRIPT void OnAudioDrainCompleted(
146       MediaDataDecoder::DecodedData&& aResults);
147   MOZ_CAN_RUN_SCRIPT void OnAudioDrainFailed(const MediaResult& aError);
148 
149   void ShutdownDecoder();
150 
151   MOZ_CAN_RUN_SCRIPT void FinishDecode();
152   MOZ_CAN_RUN_SCRIPT void AllocateBuffer();
153   MOZ_CAN_RUN_SCRIPT void CallbackTheResult();
154 
Cleanup()155   void Cleanup() {
156     MOZ_ASSERT(NS_IsMainThread());
157     JS_free(nullptr, mBuffer);
158     if (mTrackDemuxer) {
159       mTrackDemuxer->BreakCycles();
160     }
161     mTrackDemuxer = nullptr;
162     mDemuxer = nullptr;
163     mPSupervisorTaskQueue = nullptr;
164     mPDecoderTaskQueue = nullptr;
165   }
166 
167  private:
168   MediaContainerType mContainerType;
169   uint8_t* mBuffer;
170   const uint32_t mLength;
171   const uint32_t mBatchSize;
172   WebAudioDecodeJob& mDecodeJob;
173   PhaseEnum mPhase;
174   RefPtr<TaskQueue> mPSupervisorTaskQueue;
175   RefPtr<TaskQueue> mPDecoderTaskQueue;
176   RefPtr<MediaDataDemuxer> mDemuxer;
177   RefPtr<MediaTrackDemuxer> mTrackDemuxer;
178   RefPtr<MediaDataDecoder> mDecoder;
179   nsTArray<RefPtr<MediaRawData>> mRawSamples;
180   MediaInfo mMediaInfo;
181   MediaQueue<AudioData> mAudioQueue;
182   RefPtr<AbstractThread> mMainThread;
183 };
184 
185 NS_IMETHODIMP
Run()186 MediaDecodeTask::Run() {
187   switch (mPhase) {
188     case PhaseEnum::Decode:
189       Decode();
190       break;
191     case PhaseEnum::AllocateBuffer:
192       AllocateBuffer();
193       break;
194     case PhaseEnum::Done:
195       break;
196   }
197 
198   return NS_OK;
199 }
200 
Init()201 bool MediaDecodeTask::Init() {
202   MOZ_ASSERT(NS_IsMainThread());
203 
204   RefPtr<BufferMediaResource> resource =
205       new BufferMediaResource(static_cast<uint8_t*>(mBuffer), mLength);
206 
207   mMainThread = mDecodeJob.mContext->GetOwnerGlobal()->AbstractMainThreadFor(
208       TaskCategory::Other);
209 
210   mPSupervisorTaskQueue =
211       new TaskQueue(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
212                     "MediaBufferDecoder::mPSupervisorTaskQueue");
213   mPDecoderTaskQueue =
214       new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
215                     "MediaBufferDecoder::mPDecoderTaskQueue");
216 
217   // If you change this list to add support for new decoders, please consider
218   // updating HTMLMediaElement::CreateDecoder as well.
219   mDemuxer = DecoderTraits::CreateDemuxer(mContainerType, resource);
220   if (!mDemuxer) {
221     return false;
222   }
223 
224   return true;
225 }
226 
227 class AutoResampler final {
228  public:
AutoResampler()229   AutoResampler() : mResampler(nullptr) {}
~AutoResampler()230   ~AutoResampler() {
231     if (mResampler) {
232       speex_resampler_destroy(mResampler);
233     }
234   }
operator SpeexResamplerState*() const235   operator SpeexResamplerState*() const {
236     MOZ_ASSERT(mResampler);
237     return mResampler;
238   }
operator =(SpeexResamplerState * aResampler)239   void operator=(SpeexResamplerState* aResampler) { mResampler = aResampler; }
240 
241  private:
242   SpeexResamplerState* mResampler;
243 };
244 
Decode()245 void MediaDecodeTask::Decode() {
246   MOZ_ASSERT(OnPSupervisorTaskQueue());
247 
248   mDemuxer->Init()->Then(PSupervisorTaskQueue(), __func__, this,
249                          &MediaDecodeTask::OnInitDemuxerCompleted,
250                          &MediaDecodeTask::OnInitDemuxerFailed);
251 }
252 
OnInitDemuxerCompleted()253 void MediaDecodeTask::OnInitDemuxerCompleted() {
254   MOZ_ASSERT(OnPSupervisorTaskQueue());
255 
256   if (!!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack)) {
257     mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
258     if (!mTrackDemuxer) {
259       LOG("MediaDecodeTask: Could not get a track demuxer.");
260       ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
261       return;
262     }
263 
264     RefPtr<PDMFactory> platform = new PDMFactory();
265     UniquePtr<TrackInfo> audioInfo = mTrackDemuxer->GetInfo();
266     // We actively ignore audio tracks that we know we can't play.
267     if (audioInfo && audioInfo->IsValid() &&
268         platform->SupportsMimeType(audioInfo->mMimeType)) {
269       mMediaInfo.mAudio = *audioInfo->GetAsAudioInfo();
270     }
271   }
272 
273   RefPtr<PDMFactory> pdm = new PDMFactory();
274   pdm->CreateDecoder(
275          {*mMediaInfo.mAudio.GetAsAudioInfo(), TrackInfo::kAudioTrack})
276       ->Then(PSupervisorTaskQueue(), __func__, this,
277              &MediaDecodeTask::OnCreateDecoderCompleted,
278              &MediaDecodeTask::OnCreateDecoderFailed);
279 }
280 
OnCreateDecoderCompleted(RefPtr<MediaDataDecoder> aDecoder)281 void MediaDecodeTask::OnCreateDecoderCompleted(
282     RefPtr<MediaDataDecoder> aDecoder) {
283   MOZ_ASSERT(OnPSupervisorTaskQueue());
284 
285   mDecoder = new MediaDataDecoderProxy(aDecoder.forget(),
286                                        do_AddRef(mPDecoderTaskQueue.get()));
287   InitDecoder();
288 }
289 
OnCreateDecoderFailed(const MediaResult & aError)290 void MediaDecodeTask::OnCreateDecoderFailed(const MediaResult& aError) {
291   MOZ_ASSERT(OnPSupervisorTaskQueue());
292 
293   LOG("MediaDecodeTask: Could not create a decoder.");
294   ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
295 }
296 
OnInitDemuxerFailed(const MediaResult & aError)297 void MediaDecodeTask::OnInitDemuxerFailed(const MediaResult& aError) {
298   MOZ_ASSERT(OnPSupervisorTaskQueue());
299 
300   LOG("MediaDecodeTask: Could not initialize the demuxer.");
301   ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
302 }
303 
InitDecoder()304 void MediaDecodeTask::InitDecoder() {
305   MOZ_ASSERT(OnPSupervisorTaskQueue());
306 
307   mDecoder->Init()->Then(PSupervisorTaskQueue(), __func__, this,
308                          &MediaDecodeTask::OnInitDecoderCompleted,
309                          &MediaDecodeTask::OnInitDecoderFailed);
310 }
311 
OnInitDecoderCompleted()312 void MediaDecodeTask::OnInitDecoderCompleted() {
313   MOZ_ASSERT(OnPSupervisorTaskQueue());
314 
315   DoDemux();
316 }
317 
OnInitDecoderFailed()318 void MediaDecodeTask::OnInitDecoderFailed() {
319   MOZ_ASSERT(OnPSupervisorTaskQueue());
320 
321   ShutdownDecoder();
322   LOG("MediaDecodeTask: Could not initialize the decoder");
323   ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
324 }
325 
DoDemux()326 void MediaDecodeTask::DoDemux() {
327   MOZ_ASSERT(OnPSupervisorTaskQueue());
328 
329   mTrackDemuxer->GetSamples(mBatchSize)
330       ->Then(PSupervisorTaskQueue(), __func__, this,
331              &MediaDecodeTask::OnAudioDemuxCompleted,
332              &MediaDecodeTask::OnAudioDemuxFailed);
333 }
334 
OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)335 void MediaDecodeTask::OnAudioDemuxCompleted(
336     RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
337   MOZ_ASSERT(OnPSupervisorTaskQueue());
338 
339   mRawSamples.AppendElements(aSamples->GetSamples());
340 
341   DoDemux();
342 }
343 
OnAudioDemuxFailed(const MediaResult & aError)344 void MediaDecodeTask::OnAudioDemuxFailed(const MediaResult& aError) {
345   MOZ_ASSERT(OnPSupervisorTaskQueue());
346 
347   if (aError.Code() == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
348     DoDecode();
349   } else {
350     ShutdownDecoder();
351     LOG("MediaDecodeTask: Audio demux failed");
352     ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
353   }
354 }
355 
DoDecode()356 void MediaDecodeTask::DoDecode() {
357   MOZ_ASSERT(OnPSupervisorTaskQueue());
358 
359   if (mRawSamples.IsEmpty()) {
360     DoDrain();
361     return;
362   }
363 
364   if (mBatchSize > 1 && mDecoder->CanDecodeBatch()) {
365     nsTArray<RefPtr<MediaRawData>> rawSampleBatch;
366     const int batchSize = std::min((unsigned long)mBatchSize,
367                                    (unsigned long)mRawSamples.Length());
368     for (int i = 0; i < batchSize; ++i) {
369       rawSampleBatch.AppendElement(std::move(mRawSamples[i]));
370     }
371 
372     mDecoder->DecodeBatch(std::move(rawSampleBatch))
373         ->Then(PSupervisorTaskQueue(), __func__, this,
374                &MediaDecodeTask::OnAudioDecodeCompleted,
375                &MediaDecodeTask::OnAudioDecodeFailed);
376 
377     mRawSamples.RemoveElementsAt(0, batchSize);
378   } else {
379     RefPtr<MediaRawData> sample = std::move(mRawSamples[0]);
380 
381     mDecoder->Decode(sample)->Then(PSupervisorTaskQueue(), __func__, this,
382                                    &MediaDecodeTask::OnAudioDecodeCompleted,
383                                    &MediaDecodeTask::OnAudioDecodeFailed);
384 
385     mRawSamples.RemoveElementAt(0);
386   }
387 }
388 
OnAudioDecodeCompleted(MediaDataDecoder::DecodedData && aResults)389 void MediaDecodeTask::OnAudioDecodeCompleted(
390     MediaDataDecoder::DecodedData&& aResults) {
391   MOZ_ASSERT(OnPSupervisorTaskQueue());
392 
393   for (auto&& sample : aResults) {
394     MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
395     RefPtr<AudioData> audioData = sample->As<AudioData>();
396 
397     mMediaInfo.mAudio.mRate = audioData->mRate;
398     mMediaInfo.mAudio.mChannels = audioData->mChannels;
399 
400     mAudioQueue.Push(audioData.forget());
401   }
402 
403   DoDecode();
404 }
405 
OnAudioDecodeFailed(const MediaResult & aError)406 void MediaDecodeTask::OnAudioDecodeFailed(const MediaResult& aError) {
407   MOZ_ASSERT(OnPSupervisorTaskQueue());
408 
409   ShutdownDecoder();
410   LOG("MediaDecodeTask: decode audio failed.");
411   ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
412 }
413 
DoDrain()414 void MediaDecodeTask::DoDrain() {
415   MOZ_ASSERT(OnPSupervisorTaskQueue());
416 
417   mDecoder->Drain()->Then(PSupervisorTaskQueue(), __func__, this,
418                           &MediaDecodeTask::OnAudioDrainCompleted,
419                           &MediaDecodeTask::OnAudioDrainFailed);
420 }
421 
OnAudioDrainCompleted(MediaDataDecoder::DecodedData && aResults)422 void MediaDecodeTask::OnAudioDrainCompleted(
423     MediaDataDecoder::DecodedData&& aResults) {
424   MOZ_ASSERT(OnPSupervisorTaskQueue());
425 
426   if (aResults.IsEmpty()) {
427     FinishDecode();
428     return;
429   }
430 
431   for (auto&& sample : aResults) {
432     MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
433     RefPtr<AudioData> audioData = sample->As<AudioData>();
434 
435     mAudioQueue.Push(audioData.forget());
436   }
437   DoDrain();
438 }
439 
OnAudioDrainFailed(const MediaResult & aError)440 void MediaDecodeTask::OnAudioDrainFailed(const MediaResult& aError) {
441   MOZ_ASSERT(OnPSupervisorTaskQueue());
442 
443   ShutdownDecoder();
444   LOG("MediaDecodeTask: Drain audio failed");
445   ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
446 }
447 
ShutdownDecoder()448 void MediaDecodeTask::ShutdownDecoder() {
449   MOZ_ASSERT(OnPSupervisorTaskQueue());
450 
451   if (!mDecoder) {
452     return;
453   }
454 
455   RefPtr<MediaDecodeTask> self = this;
456   mDecoder->Shutdown();
457   mDecoder = nullptr;
458 }
459 
FinishDecode()460 void MediaDecodeTask::FinishDecode() {
461   MOZ_ASSERT(OnPSupervisorTaskQueue());
462 
463   ShutdownDecoder();
464 
465   uint32_t frameCount = mAudioQueue.AudioFramesCount();
466   uint32_t channelCount = mMediaInfo.mAudio.mChannels;
467   uint32_t sampleRate = mMediaInfo.mAudio.mRate;
468 
469   if (!frameCount || !channelCount || !sampleRate) {
470     LOG("MediaDecodeTask: invalid content frame count, channel count or "
471         "sample-rate");
472     ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
473     return;
474   }
475 
476   const uint32_t destSampleRate = mDecodeJob.mContext->SampleRate();
477   AutoResampler resampler;
478 
479   uint32_t resampledFrames = frameCount;
480   if (sampleRate != destSampleRate) {
481     resampledFrames = static_cast<uint32_t>(
482         static_cast<uint64_t>(destSampleRate) *
483         static_cast<uint64_t>(frameCount) / static_cast<uint64_t>(sampleRate));
484 
485     resampler = speex_resampler_init(channelCount, sampleRate, destSampleRate,
486                                      SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr);
487     speex_resampler_skip_zeros(resampler);
488     resampledFrames += speex_resampler_get_output_latency(resampler);
489   }
490 
491   // Allocate contiguous channel buffers.  Note that if we end up resampling,
492   // we may write fewer bytes than mResampledFrames to the output buffer, in
493   // which case writeIndex will tell us how many valid samples we have.
494   mDecodeJob.mBuffer.mChannelData.SetLength(channelCount);
495 #if AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_FLOAT32
496   // This buffer has separate channel arrays that could be transferred to
497   // JS::NewArrayBufferWithContents(), but AudioBuffer::RestoreJSChannelData()
498   // does not yet take advantage of this.
499   RefPtr<ThreadSharedFloatArrayBufferList> buffer =
500       ThreadSharedFloatArrayBufferList::Create(channelCount, resampledFrames,
501                                                fallible);
502   if (!buffer) {
503     LOG("MediaDecodeTask: Could not create final buffer (f32)");
504     ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
505     return;
506   }
507   for (uint32_t i = 0; i < channelCount; ++i) {
508     mDecodeJob.mBuffer.mChannelData[i] = buffer->GetData(i);
509   }
510 #else
511   CheckedInt<size_t> bufferSize(sizeof(AudioDataValue));
512   bufferSize *= resampledFrames;
513   bufferSize *= channelCount;
514   RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize);
515   if (!buffer) {
516     LOG("MediaDecodeTask: Could not create final buffer (i16)");
517     ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
518     return;
519   }
520   auto data = static_cast<AudioDataValue*>(floatBuffer->Data());
521   for (uint32_t i = 0; i < channelCount; ++i) {
522     mDecodeJob.mBuffer.mChannelData[i] = data;
523     data += resampledFrames;
524   }
525 #endif
526   mDecodeJob.mBuffer.mBuffer = std::move(buffer);
527   mDecodeJob.mBuffer.mVolume = 1.0f;
528   mDecodeJob.mBuffer.mBufferFormat = AUDIO_OUTPUT_FORMAT;
529 
530   uint32_t writeIndex = 0;
531   RefPtr<AudioData> audioData;
532   while ((audioData = mAudioQueue.PopFront())) {
533     if (!audioData->Frames()) {
534       // The packet contains no audio frames, skip it.
535       continue;
536     }
537     audioData->EnsureAudioBuffer();  // could lead to a copy :(
538     const AudioDataValue* bufferData =
539         static_cast<AudioDataValue*>(audioData->mAudioBuffer->Data());
540 
541     if (sampleRate != destSampleRate) {
542       const uint32_t maxOutSamples = resampledFrames - writeIndex;
543 
544       for (uint32_t i = 0; i < audioData->mChannels; ++i) {
545         uint32_t inSamples = audioData->Frames();
546         uint32_t outSamples = maxOutSamples;
547         AudioDataValue* outData =
548             mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
549             writeIndex;
550 
551         WebAudioUtils::SpeexResamplerProcess(
552             resampler, i, &bufferData[i * audioData->Frames()], &inSamples,
553             outData, &outSamples);
554 
555         if (i == audioData->mChannels - 1) {
556           writeIndex += outSamples;
557           MOZ_ASSERT(writeIndex <= resampledFrames);
558           MOZ_ASSERT(inSamples == audioData->Frames());
559         }
560       }
561     } else {
562       for (uint32_t i = 0; i < audioData->mChannels; ++i) {
563         AudioDataValue* outData =
564             mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
565             writeIndex;
566         PodCopy(outData, &bufferData[i * audioData->Frames()],
567                 audioData->Frames());
568 
569         if (i == audioData->mChannels - 1) {
570           writeIndex += audioData->Frames();
571         }
572       }
573     }
574   }
575 
576   if (sampleRate != destSampleRate) {
577     uint32_t inputLatency = speex_resampler_get_input_latency(resampler);
578     const uint32_t maxOutSamples = resampledFrames - writeIndex;
579     for (uint32_t i = 0; i < channelCount; ++i) {
580       uint32_t inSamples = inputLatency;
581       uint32_t outSamples = maxOutSamples;
582       AudioDataValue* outData =
583           mDecodeJob.mBuffer.ChannelDataForWrite<AudioDataValue>(i) +
584           writeIndex;
585 
586       WebAudioUtils::SpeexResamplerProcess(resampler, i,
587                                            (AudioDataValue*)nullptr, &inSamples,
588                                            outData, &outSamples);
589 
590       if (i == channelCount - 1) {
591         writeIndex += outSamples;
592         MOZ_ASSERT(writeIndex <= resampledFrames);
593         MOZ_ASSERT(inSamples == inputLatency);
594       }
595     }
596   }
597 
598   mDecodeJob.mBuffer.mDuration = writeIndex;
599   mPhase = PhaseEnum::AllocateBuffer;
600   mMainThread->Dispatch(do_AddRef(this));
601 }
602 
AllocateBuffer()603 void MediaDecodeTask::AllocateBuffer() {
604   MOZ_ASSERT(NS_IsMainThread());
605 
606   if (!mDecodeJob.AllocateBuffer()) {
607     LOG("MediaDecodeTask: Could not allocate final buffer");
608     ReportFailureOnMainThread(WebAudioDecodeJob::UnknownError);
609     return;
610   }
611 
612   mPhase = PhaseEnum::Done;
613   CallbackTheResult();
614 }
615 
CallbackTheResult()616 void MediaDecodeTask::CallbackTheResult() {
617   MOZ_ASSERT(NS_IsMainThread());
618 
619   Cleanup();
620 
621   // Now, we're ready to call the script back with the resulting buffer
622   mDecodeJob.OnSuccess(WebAudioDecodeJob::NoError);
623 }
624 
AllocateBuffer()625 bool WebAudioDecodeJob::AllocateBuffer() {
626   MOZ_ASSERT(!mOutput);
627   MOZ_ASSERT(NS_IsMainThread());
628 
629   // Now create the AudioBuffer
630   mOutput = AudioBuffer::Create(mContext->GetOwner(), mContext->SampleRate(),
631                                 std::move(mBuffer));
632   return mOutput != nullptr;
633 }
634 
AsyncDecodeWebAudio(const char * aContentType,uint8_t * aBuffer,uint32_t aLength,WebAudioDecodeJob & aDecodeJob)635 void AsyncDecodeWebAudio(const char* aContentType, uint8_t* aBuffer,
636                          uint32_t aLength, WebAudioDecodeJob& aDecodeJob) {
637   Maybe<MediaContainerType> containerType =
638       MakeMediaContainerType(aContentType);
639   // Do not attempt to decode the media if we were not successful at sniffing
640   // the container type.
641   if (!*aContentType || strcmp(aContentType, APPLICATION_OCTET_STREAM) == 0 ||
642       !containerType) {
643     nsCOMPtr<nsIRunnable> event =
644         new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
645                              WebAudioDecodeJob::UnknownContent);
646     JS_free(nullptr, aBuffer);
647     aDecodeJob.mContext->Dispatch(event.forget());
648     return;
649   }
650 
651   RefPtr<MediaDecodeTask> task =
652       new MediaDecodeTask(*containerType, aBuffer, aLength, aDecodeJob);
653   if (!task->Init()) {
654     nsCOMPtr<nsIRunnable> event =
655         new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
656                              WebAudioDecodeJob::UnknownError);
657     aDecodeJob.mContext->Dispatch(event.forget());
658   } else {
659     nsresult rv = task->PSupervisorTaskQueue()->Dispatch(task.forget());
660     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
661     Unused << rv;
662   }
663 }
664 
WebAudioDecodeJob(AudioContext * aContext,Promise * aPromise,DecodeSuccessCallback * aSuccessCallback,DecodeErrorCallback * aFailureCallback)665 WebAudioDecodeJob::WebAudioDecodeJob(AudioContext* aContext, Promise* aPromise,
666                                      DecodeSuccessCallback* aSuccessCallback,
667                                      DecodeErrorCallback* aFailureCallback)
668     : mContext(aContext),
669       mPromise(aPromise),
670       mSuccessCallback(aSuccessCallback),
671       mFailureCallback(aFailureCallback) {
672   MOZ_ASSERT(aContext);
673   MOZ_ASSERT(NS_IsMainThread());
674   MOZ_COUNT_CTOR(WebAudioDecodeJob);
675 }
676 
~WebAudioDecodeJob()677 WebAudioDecodeJob::~WebAudioDecodeJob() {
678   MOZ_ASSERT(NS_IsMainThread());
679   MOZ_COUNT_DTOR(WebAudioDecodeJob);
680 }
681 
OnSuccess(ErrorCode aErrorCode)682 void WebAudioDecodeJob::OnSuccess(ErrorCode aErrorCode) {
683   MOZ_ASSERT(NS_IsMainThread());
684   MOZ_ASSERT(aErrorCode == NoError);
685 
686   RefPtr<AudioBuffer> output(mOutput);
687   if (mSuccessCallback) {
688     RefPtr<DecodeSuccessCallback> callback(mSuccessCallback);
689     // Ignore errors in calling the callback, since there is not much that we
690     // can do about it here.
691     callback->Call(*output);
692   }
693   mPromise->MaybeResolve(output);
694 
695   mContext->RemoveFromDecodeQueue(this);
696 }
697 
OnFailure(ErrorCode aErrorCode)698 void WebAudioDecodeJob::OnFailure(ErrorCode aErrorCode) {
699   MOZ_ASSERT(NS_IsMainThread());
700 
701   const char* errorMessage;
702   switch (aErrorCode) {
703     case UnknownContent:
704       errorMessage =
705           "The buffer passed to decodeAudioData contains an unknown content "
706           "type.";
707       break;
708     case InvalidContent:
709       errorMessage =
710           "The buffer passed to decodeAudioData contains invalid content which "
711           "cannot be decoded successfully.";
712       break;
713     case NoAudio:
714       errorMessage =
715           "The buffer passed to decodeAudioData does not contain any audio.";
716       break;
717     case NoError:
718       MOZ_FALLTHROUGH_ASSERT("Who passed NoError to OnFailure?");
719       // Fall through to get some sort of a sane error message if this actually
720       // happens at runtime.
721     case UnknownError:
722       [[fallthrough]];
723     default:
724       errorMessage =
725           "An unknown error occurred while processing decodeAudioData.";
726       break;
727   }
728 
729   // Ignore errors in calling the callback, since there is not much that we can
730   // do about it here.
731   nsAutoCString errorString(errorMessage);
732   if (mFailureCallback) {
733     RefPtr<DOMException> exception = DOMException::Create(
734         NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR, errorString);
735     RefPtr<DecodeErrorCallback> callback(mFailureCallback);
736     callback->Call(*exception);
737   }
738 
739   mPromise->MaybeRejectWithEncodingError(errorString);
740 
741   mContext->RemoveFromDecodeQueue(this);
742 }
743 
SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const744 size_t WebAudioDecodeJob::SizeOfExcludingThis(
745     MallocSizeOf aMallocSizeOf) const {
746   size_t amount = 0;
747   if (mSuccessCallback) {
748     amount += mSuccessCallback->SizeOfIncludingThis(aMallocSizeOf);
749   }
750   if (mFailureCallback) {
751     amount += mFailureCallback->SizeOfIncludingThis(aMallocSizeOf);
752   }
753   if (mOutput) {
754     amount += mOutput->SizeOfIncludingThis(aMallocSizeOf);
755   }
756   amount += mBuffer.SizeOfExcludingThis(aMallocSizeOf, false);
757   return amount;
758 }
759 
SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const760 size_t WebAudioDecodeJob::SizeOfIncludingThis(
761     MallocSizeOf aMallocSizeOf) const {
762   return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
763 }
764 
765 }  // namespace mozilla
766