1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaFormatReader.h"
8
9 #include "AllocationPolicy.h"
10 #include "DecoderBenchmark.h"
11 #include "GeckoProfiler.h"
12 #include "MediaData.h"
13 #include "MediaInfo.h"
14 #include "VideoFrameContainer.h"
15 #include "VideoUtils.h"
16 #include "mozilla/AbstractThread.h"
17 #include "mozilla/CDMProxy.h"
18 #include "mozilla/ClearOnShutdown.h"
19 #include "mozilla/NotNull.h"
20 #include "mozilla/Preferences.h"
21 #include "mozilla/SharedThreadPool.h"
22 #include "mozilla/StaticPrefs_media.h"
23 #include "mozilla/TaskQueue.h"
24 #include "mozilla/Unused.h"
25 #include "nsContentUtils.h"
26 #include "nsPrintfCString.h"
27
28 #include <algorithm>
29 #include <map>
30 #include <queue>
31
32 using namespace mozilla::media;
33
34 static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
35 mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
36
37 #define LOG(arg, ...) \
38 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
39 __func__, ##__VA_ARGS__)
40 #define LOGV(arg, ...) \
41 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
42 __func__, ##__VA_ARGS__)
43
44 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
45
46 namespace mozilla {
47
48 typedef void* MediaDataDecoderID;
49
50 /**
51 * This class tracks shutdown promises to ensure all decoders are shut down
52 * completely before MFR continues the rest of the shutdown procedure.
53 */
54 class MediaFormatReader::ShutdownPromisePool {
55 public:
ShutdownPromisePool()56 ShutdownPromisePool()
57 : mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}
58
59 // Return a promise which will be resolved when all the tracking promises
60 // are resolved. Note no more promises should be added for tracking once
61 // this function is called.
62 RefPtr<ShutdownPromise> Shutdown();
63
64 // Track a shutdown promise.
65 void Track(RefPtr<ShutdownPromise> aPromise);
66
67 // Shut down a decoder and track its shutdown promise.
ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder)68 void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {
69 Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());
70 }
71
72 private:
73 bool mShutdown = false;
74 const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;
75 nsTHashtable<nsRefPtrHashKey<ShutdownPromise>> mPromises;
76 };
77
Shutdown()78 RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {
79 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
80 mShutdown = true;
81 if (mPromises.Count() == 0) {
82 mOnShutdownComplete->Resolve(true, __func__);
83 }
84 return mOnShutdownComplete;
85 }
86
Track(RefPtr<ShutdownPromise> aPromise)87 void MediaFormatReader::ShutdownPromisePool::Track(
88 RefPtr<ShutdownPromise> aPromise) {
89 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
90 MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));
91 mPromises.PutEntry(aPromise);
92 aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {
93 MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));
94 mPromises.RemoveEntry(aPromise);
95 if (mShutdown && mPromises.Count() == 0) {
96 mOnShutdownComplete->Resolve(true, __func__);
97 }
98 });
99 }
100
ShutdownDecoder()101 void MediaFormatReader::DecoderData::ShutdownDecoder() {
102 MOZ_ASSERT(mOwner->OnTaskQueue());
103
104 MutexAutoLock lock(mMutex);
105
106 if (!mDecoder) {
107 // No decoder to shut down.
108 return;
109 }
110
111 if (mFlushing) {
112 // Flush is is in action. Shutdown will be initiated after flush completes.
113 MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);
114 mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));
115 // The order of decoder creation and shutdown is handled by LocalAllocPolicy
116 // and ShutdownPromisePool. MFR can now reset these members to a fresh state
117 // and be ready to create new decoders again without explicitly waiting for
118 // flush/shutdown to complete.
119 mShutdownPromise = nullptr;
120 mFlushing = false;
121 } else {
122 // No flush is in action. We can shut down the decoder now.
123 mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());
124 }
125
126 // mShutdownPromisePool will handle the order of decoder shutdown so
127 // we can forget mDecoder and be ready to create a new one.
128 mDecoder = nullptr;
129 mDescription = NS_LITERAL_CSTRING("shutdown");
130 mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA
131 ? TrackType::kAudioTrack
132 : TrackType::kVideoTrack);
133 }
134
Flush()135 void MediaFormatReader::DecoderData::Flush() {
136 AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);
137 MOZ_ASSERT(mOwner->OnTaskQueue());
138
139 if (mFlushing || mFlushed) {
140 // Flush still pending or already flushed, nothing more to do.
141 return;
142 }
143 mDecodeRequest.DisconnectIfExists();
144 mDrainRequest.DisconnectIfExists();
145 mDrainState = DrainState::None;
146 CancelWaitingForKey();
147 mOutput.Clear();
148 mNumSamplesInput = 0;
149 mNumSamplesOutput = 0;
150 mSizeOfQueue = 0;
151 if (mDecoder) {
152 TrackType type = mType == MediaData::Type::AUDIO_DATA
153 ? TrackType::kAudioTrack
154 : TrackType::kVideoTrack;
155 mFlushing = true;
156 MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);
157 mShutdownPromise = new SharedShutdownPromiseHolder();
158 RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;
159 RefPtr<MediaDataDecoder> d = mDecoder;
160 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
161 "flushing", DDNoValue{});
162 mDecoder->Flush()->Then(
163 mOwner->OwnerThread(), __func__,
164 [type, this, p, d]() {
165 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
166 MEDIA_PLAYBACK);
167 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
168 "flushed", DDNoValue{});
169 if (!p->IsEmpty()) {
170 // Shutdown happened before flush completes.
171 // Let's continue to shut down the decoder. Note
172 // we don't access |this| because this decoder
173 // is no longer managed by MFR::DecoderData.
174 d->Shutdown()->ChainTo(p->Steal(), __func__);
175 return;
176 }
177 mFlushing = false;
178 mShutdownPromise = nullptr;
179 mOwner->ScheduleUpdate(type);
180 },
181 [type, this, p, d](const MediaResult& aError) {
182 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
183 MEDIA_PLAYBACK);
184 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
185 "flush_error", aError);
186 if (!p->IsEmpty()) {
187 d->Shutdown()->ChainTo(p->Steal(), __func__);
188 return;
189 }
190 mFlushing = false;
191 mShutdownPromise = nullptr;
192 mOwner->NotifyError(type, aError);
193 });
194 }
195 mFlushed = true;
196 }
197
198 class MediaFormatReader::DecoderFactory {
199 using InitPromise = MediaDataDecoder::InitPromise;
200 using TokenPromise = AllocPolicy::Promise;
201 using Token = AllocPolicy::Token;
202
203 public:
DecoderFactory(MediaFormatReader * aOwner)204 explicit DecoderFactory(MediaFormatReader* aOwner)
205 : mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),
206 mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),
207 mOwner(WrapNotNull(aOwner)) {
208 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
209 this);
210 DecoderDoctorLogger::LinkParentAndChild(
211 aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);
212 }
213
~DecoderFactory()214 ~DecoderFactory() {
215 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
216 this);
217 }
218
219 void CreateDecoder(TrackType aTrack);
220
221 // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
222 // pristine state so CreateDecoder() is ready to be called again immediately.
ShutdownDecoder(TrackType aTrack)223 void ShutdownDecoder(TrackType aTrack) {
224 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
225 aTrack == TrackInfo::kVideoTrack);
226 auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
227 data.mPolicy->Cancel();
228 data.mTokenRequest.DisconnectIfExists();
229 data.mInitRequest.DisconnectIfExists();
230 if (data.mDecoder) {
231 mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());
232 }
233 data.mStage = Stage::None;
234 MOZ_ASSERT(!data.mToken);
235 }
236
237 private:
238 enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };
239
240 struct Data {
Datamozilla::MediaFormatReader::DecoderFactory::Data241 Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)
242 : mOwnerData(aOwnerData),
243 mTrack(aTrack),
244 mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}
245 DecoderData& mOwnerData;
246 const TrackType mTrack;
247 RefPtr<SingleAllocPolicy> mPolicy;
248 Stage mStage = Stage::None;
249 RefPtr<Token> mToken;
250 RefPtr<MediaDataDecoder> mDecoder;
251 MozPromiseRequestHolder<TokenPromise> mTokenRequest;
252 MozPromiseRequestHolder<InitPromise> mInitRequest;
253 } mAudio, mVideo;
254
255 void RunStage(Data& aData);
256 MediaResult DoCreateDecoder(Data& aData);
257 void DoInitDecoder(Data& aData);
258
259 // guaranteed to be valid by the owner.
260 const NotNull<MediaFormatReader*> mOwner;
261 };
262
CreateDecoder(TrackType aTrack)263 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {
264 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
265 aTrack == TrackInfo::kVideoTrack);
266 RunStage(aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo);
267 }
268
RunStage(Data & aData)269 void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {
270 switch (aData.mStage) {
271 case Stage::None: {
272 MOZ_ASSERT(!aData.mToken);
273 aData.mPolicy->Alloc()
274 ->Then(
275 mOwner->OwnerThread(), __func__,
276 [this, &aData](RefPtr<Token> aToken) {
277 aData.mTokenRequest.Complete();
278 aData.mToken = std::move(aToken);
279 aData.mStage = Stage::CreateDecoder;
280 RunStage(aData);
281 },
282 [&aData]() {
283 aData.mTokenRequest.Complete();
284 aData.mStage = Stage::None;
285 })
286 ->Track(aData.mTokenRequest);
287 aData.mStage = Stage::WaitForToken;
288 break;
289 }
290
291 case Stage::WaitForToken: {
292 MOZ_ASSERT(!aData.mToken);
293 MOZ_ASSERT(aData.mTokenRequest.Exists());
294 break;
295 }
296
297 case Stage::CreateDecoder: {
298 MOZ_ASSERT(aData.mToken);
299 MOZ_ASSERT(!aData.mDecoder);
300 MOZ_ASSERT(!aData.mInitRequest.Exists());
301
302 MediaResult rv = DoCreateDecoder(aData);
303 if (NS_FAILED(rv)) {
304 NS_WARNING("Error constructing decoders");
305 aData.mToken = nullptr;
306 aData.mStage = Stage::None;
307 aData.mOwnerData.mDescription = rv.Description();
308 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
309 "create_decoder_error", rv);
310 mOwner->NotifyError(aData.mTrack, rv);
311 return;
312 }
313
314 aData.mDecoder =
315 new AllocationWrapper(aData.mDecoder.forget(), aData.mToken.forget());
316 DecoderDoctorLogger::LinkParentAndChild(
317 aData.mDecoder.get(), "decoder", "MediaFormatReader::DecoderFactory",
318 this);
319
320 DoInitDecoder(aData);
321 aData.mStage = Stage::WaitForInit;
322 break;
323 }
324
325 case Stage::WaitForInit: {
326 MOZ_ASSERT(aData.mDecoder);
327 MOZ_ASSERT(aData.mInitRequest.Exists());
328 break;
329 }
330 }
331 }
332
DoCreateDecoder(Data & aData)333 MediaResult MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {
334 AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);
335 auto& ownerData = aData.mOwnerData;
336 auto& decoder = mOwner->GetDecoderData(aData.mTrack);
337 auto& platform =
338 decoder.IsEncrypted() ? mOwner->mEncryptedPlatform : mOwner->mPlatform;
339
340 if (!platform) {
341 platform = new PDMFactory();
342 if (decoder.IsEncrypted()) {
343 MOZ_ASSERT(mOwner->mCDMProxy);
344 platform->SetCDMProxy(mOwner->mCDMProxy);
345 }
346 }
347
348 // result may not be updated by PDMFactory::CreateDecoder, as such it must be
349 // initialized to a fatal error by default.
350 MediaResult result =
351 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
352 nsPrintfCString("error creating %s decoder",
353 TrackTypeToStr(aData.mTrack)));
354
355 switch (aData.mTrack) {
356 case TrackInfo::kAudioTrack: {
357 aData.mDecoder = platform->CreateDecoder(
358 {*ownerData.GetCurrentInfo()->GetAsAudioInfo(), ownerData.mTaskQueue,
359 mOwner->mCrashHelper,
360 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
361 &result, TrackInfo::kAudioTrack,
362 &mOwner->OnTrackWaitingForKeyProducer()});
363 break;
364 }
365
366 case TrackType::kVideoTrack: {
367 // Decoders use the layers backend to decide if they can use hardware
368 // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
369 using Option = CreateDecoderParams::Option;
370 using OptionSet = CreateDecoderParams::OptionSet;
371
372 aData.mDecoder = platform->CreateDecoder(
373 {*ownerData.GetCurrentInfo()->GetAsVideoInfo(), ownerData.mTaskQueue,
374 mOwner->mKnowsCompositor, mOwner->GetImageContainer(),
375 mOwner->mCrashHelper,
376 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
377 &result, TrackType::kVideoTrack,
378 &mOwner->OnTrackWaitingForKeyProducer(),
379 CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),
380 OptionSet(ownerData.mHardwareDecodingDisabled
381 ? Option::HardwareDecoderNotAllowed
382 : Option::Default)});
383 break;
384 }
385
386 default:
387 break;
388 }
389
390 if (aData.mDecoder) {
391 return NS_OK;
392 }
393
394 MOZ_RELEASE_ASSERT(NS_FAILED(result), "PDM returned an invalid error code");
395
396 return result;
397 }
398
DoInitDecoder(Data & aData)399 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {
400 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);
401 auto& ownerData = aData.mOwnerData;
402
403 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
404 "initialize_decoder", DDNoValue{});
405 aData.mDecoder->Init()
406 ->Then(
407 mOwner->OwnerThread(), __func__,
408 [this, &aData, &ownerData](TrackType aTrack) {
409 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
410 MEDIA_PLAYBACK);
411 aData.mInitRequest.Complete();
412 aData.mStage = Stage::None;
413 MutexAutoLock lock(ownerData.mMutex);
414 ownerData.mDecoder = std::move(aData.mDecoder);
415 ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
416 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
417 DDLogCategory::Log, "decoder_initialized", DDNoValue{});
418 DecoderDoctorLogger::LinkParentAndChild(
419 "MediaFormatReader::DecoderData", &ownerData, "decoder",
420 ownerData.mDecoder.get());
421 mOwner->SetVideoDecodeThreshold();
422 mOwner->ScheduleUpdate(aTrack);
423 if (aTrack == TrackInfo::kVideoTrack) {
424 DecoderBenchmark::CheckVersion(
425 ownerData.GetCurrentInfo()->mMimeType);
426 }
427 },
428 [this, &aData, &ownerData](const MediaResult& aError) {
429 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
430 MEDIA_PLAYBACK);
431 aData.mInitRequest.Complete();
432 MOZ_RELEASE_ASSERT(!ownerData.mDecoder,
433 "Can't have a decoder already set");
434 aData.mStage = Stage::None;
435 mOwner->mShutdownPromisePool->ShutdownDecoder(
436 aData.mDecoder.forget());
437 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
438 DDLogCategory::Log, "initialize_decoder_error", aError);
439 mOwner->NotifyError(aData.mTrack, aError);
440 })
441 ->Track(aData.mInitRequest);
442 }
443
444 // DemuxerProxy ensures that the original main demuxer is only ever accessed
445 // via its own dedicated task queue.
446 // This ensure that the reader's taskqueue will never blocked while a demuxer
447 // is itself blocked attempting to access the MediaCache or the MediaResource.
448 class MediaFormatReader::DemuxerProxy {
449 using TrackType = TrackInfo::TrackType;
450 class Wrapper;
451
452 public:
DemuxerProxy(MediaDataDemuxer * aDemuxer)453 explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)
454 : mTaskQueue(
455 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
456 "DemuxerProxy::mTaskQueue")),
457 mData(new Data(aDemuxer)) {
458 MOZ_COUNT_CTOR(DemuxerProxy);
459 }
460
MOZ_COUNTED_DTOR(DemuxerProxy)461 MOZ_COUNTED_DTOR(DemuxerProxy)
462
463 RefPtr<ShutdownPromise> Shutdown() {
464 RefPtr<Data> data = std::move(mData);
465 return InvokeAsync(mTaskQueue, __func__, [data]() {
466 // We need to clear our reference to the demuxer now. So that in the event
467 // the init promise wasn't resolved, such as what can happen with the
468 // mediasource demuxer that is waiting on more data, it will force the
469 // init promise to be rejected.
470 data->mDemuxer = nullptr;
471 data->mAudioDemuxer = nullptr;
472 data->mVideoDemuxer = nullptr;
473 return ShutdownPromise::CreateAndResolve(true, __func__);
474 });
475 }
476
477 RefPtr<MediaDataDemuxer::InitPromise> Init();
478
GetTrackDemuxer(TrackType aTrack,uint32_t aTrackNumber)479 Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {
480 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
481
482 switch (aTrack) {
483 case TrackInfo::kAudioTrack:
484 return mData->mAudioDemuxer;
485 case TrackInfo::kVideoTrack:
486 return mData->mVideoDemuxer;
487 default:
488 return nullptr;
489 }
490 }
491
GetNumberTracks(TrackType aTrack) const492 uint32_t GetNumberTracks(TrackType aTrack) const {
493 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
494
495 switch (aTrack) {
496 case TrackInfo::kAudioTrack:
497 return mData->mNumAudioTrack;
498 case TrackInfo::kVideoTrack:
499 return mData->mNumVideoTrack;
500 default:
501 return 0;
502 }
503 }
504
IsSeekable() const505 bool IsSeekable() const {
506 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
507
508 return mData->mSeekable;
509 }
510
IsSeekableOnlyInBufferedRanges() const511 bool IsSeekableOnlyInBufferedRanges() const {
512 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
513
514 return mData->mSeekableOnlyInBufferedRange;
515 }
516
GetCrypto() const517 UniquePtr<EncryptionInfo> GetCrypto() const {
518 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
519
520 if (!mData->mCrypto) {
521 return nullptr;
522 }
523 auto crypto = MakeUnique<EncryptionInfo>();
524 *crypto = *mData->mCrypto;
525 return crypto;
526 }
527
528 RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();
529
ShouldComputeStartTime() const530 bool ShouldComputeStartTime() const {
531 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
532
533 return mData->mShouldComputeStartTime;
534 }
535
536 private:
537 const RefPtr<TaskQueue> mTaskQueue;
538 struct Data {
539 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)
540
Datamozilla::MediaFormatReader::DemuxerProxy::Data541 explicit Data(MediaDataDemuxer* aDemuxer)
542 : mInitDone(false), mDemuxer(aDemuxer) {}
543
544 Atomic<bool> mInitDone;
545 // Only ever accessed over mTaskQueue once.
546 RefPtr<MediaDataDemuxer> mDemuxer;
547 // Only accessed once InitPromise has been resolved and immutable after.
548 // So we can safely access them without the use of the mutex.
549 uint32_t mNumAudioTrack = 0;
550 RefPtr<Wrapper> mAudioDemuxer;
551 uint32_t mNumVideoTrack = 0;
552 RefPtr<Wrapper> mVideoDemuxer;
553 bool mSeekable = false;
554 bool mSeekableOnlyInBufferedRange = false;
555 bool mShouldComputeStartTime = true;
556 UniquePtr<EncryptionInfo> mCrypto;
557
558 private:
559 ~Data() = default;
560 };
561 RefPtr<Data> mData;
562 };
563
564 class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {
565 public:
Wrapper(MediaTrackDemuxer * aTrackDemuxer,TaskQueue * aTaskQueue)566 Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)
567 : mMutex("TrackDemuxer Mutex"),
568 mTaskQueue(aTaskQueue),
569 mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),
570 mInfo(aTrackDemuxer->GetInfo()),
571 mTrackDemuxer(aTrackDemuxer) {
572 DecoderDoctorLogger::LogConstructionAndBase(
573 "MediaFormatReader::DemuxerProxy::Wrapper", this,
574 static_cast<const MediaTrackDemuxer*>(this));
575 DecoderDoctorLogger::LinkParentAndChild(
576 "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
577 aTrackDemuxer);
578 }
579
GetInfo() const580 UniquePtr<TrackInfo> GetInfo() const override {
581 if (!mInfo) {
582 return nullptr;
583 }
584 return mInfo->Clone();
585 }
586
Seek(const TimeUnit & aTime)587 RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {
588 RefPtr<Wrapper> self = this;
589 return InvokeAsync(
590 mTaskQueue, __func__,
591 [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })
592 ->Then(
593 mTaskQueue, __func__,
594 [self](const TimeUnit& aTime) {
595 self->UpdateRandomAccessPoint();
596 return SeekPromise::CreateAndResolve(aTime, __func__);
597 },
598 [self](const MediaResult& aError) {
599 self->UpdateRandomAccessPoint();
600 return SeekPromise::CreateAndReject(aError, __func__);
601 });
602 }
603
GetSamples(int32_t aNumSamples)604 RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {
605 RefPtr<Wrapper> self = this;
606 return InvokeAsync(mTaskQueue, __func__,
607 [self, aNumSamples]() {
608 return self->mTrackDemuxer->GetSamples(aNumSamples);
609 })
610 ->Then(
611 mTaskQueue, __func__,
612 [self](RefPtr<SamplesHolder> aSamples) {
613 self->UpdateRandomAccessPoint();
614 return SamplesPromise::CreateAndResolve(aSamples.forget(),
615 __func__);
616 },
617 [self](const MediaResult& aError) {
618 self->UpdateRandomAccessPoint();
619 return SamplesPromise::CreateAndReject(aError, __func__);
620 });
621 }
622
GetSamplesMayBlock() const623 bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }
624
Reset()625 void Reset() override {
626 RefPtr<Wrapper> self = this;
627 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
628 "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
629 [self]() { self->mTrackDemuxer->Reset(); }));
630 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
631 Unused << rv;
632 }
633
GetNextRandomAccessPoint(TimeUnit * aTime)634 nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {
635 MutexAutoLock lock(mMutex);
636 if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {
637 *aTime = mNextRandomAccessPoint;
638 }
639 return mNextRandomAccessPointResult;
640 }
641
SkipToNextRandomAccessPoint(const TimeUnit & aTimeThreshold)642 RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
643 const TimeUnit& aTimeThreshold) override {
644 RefPtr<Wrapper> self = this;
645 return InvokeAsync(
646 mTaskQueue, __func__,
647 [self, aTimeThreshold]() {
648 return self->mTrackDemuxer->SkipToNextRandomAccessPoint(
649 aTimeThreshold);
650 })
651 ->Then(
652 mTaskQueue, __func__,
653 [self](uint32_t aVal) {
654 self->UpdateRandomAccessPoint();
655 return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);
656 },
657 [self](const SkipFailureHolder& aError) {
658 self->UpdateRandomAccessPoint();
659 return SkipAccessPointPromise::CreateAndReject(aError, __func__);
660 });
661 }
662
GetBuffered()663 TimeIntervals GetBuffered() override {
664 MutexAutoLock lock(mMutex);
665 return mBuffered;
666 }
667
BreakCycles()668 void BreakCycles() override {}
669
670 private:
671 Mutex mMutex;
672 const RefPtr<TaskQueue> mTaskQueue;
673 const bool mGetSamplesMayBlock;
674 const UniquePtr<TrackInfo> mInfo;
675 // mTrackDemuxer is only ever accessed on demuxer's task queue.
676 RefPtr<MediaTrackDemuxer> mTrackDemuxer;
677 // All following members are protected by mMutex
678 nsresult mNextRandomAccessPointResult = NS_OK;
679 TimeUnit mNextRandomAccessPoint;
680 TimeIntervals mBuffered;
681 friend class DemuxerProxy;
682
~Wrapper()683 ~Wrapper() {
684 RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);
685 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
686 "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
687 [trackDemuxer]() { trackDemuxer->BreakCycles(); }));
688 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
689 Unused << rv;
690 DecoderDoctorLogger::LogDestruction(
691 "MediaFormatReader::DemuxerProxy::Wrapper", this);
692 }
693
UpdateRandomAccessPoint()694 void UpdateRandomAccessPoint() {
695 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
696 if (!mTrackDemuxer) {
697 // Detached.
698 return;
699 }
700 MutexAutoLock lock(mMutex);
701 mNextRandomAccessPointResult =
702 mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);
703 }
704
UpdateBuffered()705 void UpdateBuffered() {
706 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
707 if (!mTrackDemuxer) {
708 // Detached.
709 return;
710 }
711 MutexAutoLock lock(mMutex);
712 mBuffered = mTrackDemuxer->GetBuffered();
713 }
714 };
715
Init()716 RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {
717 AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);
718 using InitPromise = MediaDataDemuxer::InitPromise;
719
720 RefPtr<Data> data = mData;
721 RefPtr<TaskQueue> taskQueue = mTaskQueue;
722 return InvokeAsync(mTaskQueue, __func__,
723 [data, taskQueue]() {
724 if (!data->mDemuxer) {
725 return InitPromise::CreateAndReject(
726 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
727 }
728 return data->mDemuxer->Init();
729 })
730 ->Then(
731 taskQueue, __func__,
732 [data, taskQueue]() {
733 AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);
734 if (!data->mDemuxer) { // Was shutdown.
735 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
736 __func__);
737 }
738 data->mNumAudioTrack =
739 data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
740 if (data->mNumAudioTrack) {
741 RefPtr<MediaTrackDemuxer> d =
742 data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
743 if (d) {
744 RefPtr<Wrapper> wrapper =
745 new DemuxerProxy::Wrapper(d, taskQueue);
746 wrapper->UpdateBuffered();
747 data->mAudioDemuxer = wrapper;
748 DecoderDoctorLogger::LinkParentAndChild(
749 data->mDemuxer.get(), "decoder factory wrapper",
750 "MediaFormatReader::DecoderFactory::Wrapper",
751 wrapper.get());
752 }
753 }
754 data->mNumVideoTrack =
755 data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
756 if (data->mNumVideoTrack) {
757 RefPtr<MediaTrackDemuxer> d =
758 data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
759 if (d) {
760 RefPtr<Wrapper> wrapper =
761 new DemuxerProxy::Wrapper(d, taskQueue);
762 wrapper->UpdateBuffered();
763 data->mVideoDemuxer = wrapper;
764 DecoderDoctorLogger::LinkParentAndChild(
765 data->mDemuxer.get(), "decoder factory wrapper",
766 "MediaFormatReader::DecoderFactory::Wrapper",
767 wrapper.get());
768 }
769 }
770 data->mCrypto = data->mDemuxer->GetCrypto();
771 data->mSeekable = data->mDemuxer->IsSeekable();
772 data->mSeekableOnlyInBufferedRange =
773 data->mDemuxer->IsSeekableOnlyInBufferedRanges();
774 data->mShouldComputeStartTime =
775 data->mDemuxer->ShouldComputeStartTime();
776 data->mInitDone = true;
777 return InitPromise::CreateAndResolve(NS_OK, __func__);
778 },
779 [](const MediaResult& aError) {
780 return InitPromise::CreateAndReject(aError, __func__);
781 });
782 }
783
784 RefPtr<MediaFormatReader::NotifyDataArrivedPromise>
NotifyDataArrived()785 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
786 RefPtr<Data> data = mData;
787 return InvokeAsync(mTaskQueue, __func__, [data]() {
788 if (!data->mDemuxer) {
789 // Was shutdown.
790 return NotifyDataArrivedPromise::CreateAndReject(
791 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
792 }
793 data->mDemuxer->NotifyDataArrived();
794 if (data->mAudioDemuxer) {
795 data->mAudioDemuxer->UpdateBuffered();
796 }
797 if (data->mVideoDemuxer) {
798 data->mVideoDemuxer->UpdateBuffered();
799 }
800 return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);
801 });
802 }
803
MediaFormatReader(MediaFormatReaderInit & aInit,MediaDataDemuxer * aDemuxer)804 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
805 MediaDataDemuxer* aDemuxer)
806 : mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
807 "MediaFormatReader::mTaskQueue",
808 /* aSupportsTailDispatch = */ true)),
809 mAudio(this, MediaData::Type::AUDIO_DATA,
810 StaticPrefs::media_audio_max_decode_error()),
811 mVideo(this, MediaData::Type::VIDEO_DATA,
812 StaticPrefs::media_video_max_decode_error()),
813 mDemuxer(new DemuxerProxy(aDemuxer)),
814 mDemuxerInitDone(false),
815 mPendingNotifyDataArrived(false),
816 mLastReportedNumDecodedFrames(0),
817 mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),
818 mKnowsCompositor(aInit.mKnowsCompositor),
819 mInitDone(false),
820 mTrackDemuxersMayBlock(false),
821 mSeekScheduled(false),
822 mVideoFrameContainer(aInit.mVideoFrameContainer),
823 mCrashHelper(aInit.mCrashHelper),
824 mDecoderFactory(new DecoderFactory(this)),
825 mShutdownPromisePool(new ShutdownPromisePool()),
826 mBuffered(mTaskQueue, TimeIntervals(),
827 "MediaFormatReader::mBuffered (Canonical)"),
828 mFrameStats(aInit.mFrameStats),
829 mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID) {
830 MOZ_ASSERT(aDemuxer);
831 MOZ_COUNT_CTOR(MediaFormatReader);
832 DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
833 &mAudio);
834 DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
835 &mVideo);
836 DDLINKCHILD("demuxer", aDemuxer);
837 mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(
838 mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);
839 }
840
~MediaFormatReader()841 MediaFormatReader::~MediaFormatReader() {
842 MOZ_COUNT_DTOR(MediaFormatReader);
843 MOZ_ASSERT(mShutdown);
844 }
845
Shutdown()846 RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
847 MOZ_ASSERT(OnTaskQueue());
848 LOG("");
849
850 mDemuxerInitRequest.DisconnectIfExists();
851 mNotifyDataArrivedPromise.DisconnectIfExists();
852 mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
853 mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
854 mSkipRequest.DisconnectIfExists();
855 mSetCDMPromise.RejectIfExists(
856 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
857 "MediaFormatReader is shutting down"),
858 __func__);
859
860 if (mAudio.HasPromise()) {
861 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
862 }
863 if (mVideo.HasPromise()) {
864 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
865 }
866
867 if (HasAudio()) {
868 mAudio.ResetDemuxer();
869 mAudio.mTrackDemuxer->BreakCycles();
870 {
871 MutexAutoLock lock(mAudio.mMutex);
872 mAudio.mTrackDemuxer = nullptr;
873 }
874 mAudio.ResetState();
875 ShutdownDecoder(TrackInfo::kAudioTrack);
876 }
877
878 if (HasVideo()) {
879 mVideo.ResetDemuxer();
880 mVideo.mTrackDemuxer->BreakCycles();
881 {
882 MutexAutoLock lock(mVideo.mMutex);
883 mVideo.mTrackDemuxer = nullptr;
884 }
885 mVideo.ResetState();
886 ShutdownDecoder(TrackInfo::kVideoTrack);
887 }
888
889 mShutdownPromisePool->Track(mDemuxer->Shutdown());
890 mDemuxer = nullptr;
891
892 mOnTrackWaitingForKeyListener.Disconnect();
893
894 mShutdown = true;
895 return mShutdownPromisePool->Shutdown()->Then(
896 OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,
897 &MediaFormatReader::TearDownDecoders);
898 }
899
ShutdownDecoder(TrackType aTrack)900 void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {
901 LOGV("%s", TrackTypeToStr(aTrack));
902
903 // Shut down the pending decoder if any.
904 mDecoderFactory->ShutdownDecoder(aTrack);
905
906 auto& decoder = GetDecoderData(aTrack);
907 // Flush the decoder if necessary.
908 decoder.Flush();
909
910 // Shut down the decoder if any.
911 decoder.ShutdownDecoder();
912 }
913
NotifyDecoderBenchmarkStore()914 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
915 MOZ_ASSERT(OnTaskQueue());
916 if (!StaticPrefs::media_mediacapabilities_from_database()) {
917 return;
918 }
919 auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);
920 if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {
921 VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());
922 info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));
923 mOnStoreDecoderBenchmark.Notify(std::move(info));
924 }
925 }
926
TearDownDecoders()927 RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
928 if (mAudio.mTaskQueue) {
929 mAudio.mTaskQueue->BeginShutdown();
930 mAudio.mTaskQueue->AwaitShutdownAndIdle();
931 mAudio.mTaskQueue = nullptr;
932 }
933 if (mVideo.mTaskQueue) {
934 mVideo.mTaskQueue->BeginShutdown();
935 mVideo.mTaskQueue->AwaitShutdownAndIdle();
936 mVideo.mTaskQueue = nullptr;
937 }
938
939 mDecoderFactory = nullptr;
940 mPlatform = nullptr;
941 mEncryptedPlatform = nullptr;
942 mVideoFrameContainer = nullptr;
943
944 ReleaseResources();
945 mBuffered.DisconnectAll();
946 return mTaskQueue->BeginShutdown();
947 }
948
Init()949 nsresult MediaFormatReader::Init() {
950 MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
951
952 mAudio.mTaskQueue =
953 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
954 "MFR::mAudio::mTaskQueue");
955
956 mVideo.mTaskQueue =
957 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
958 "MFR::mVideo::mTaskQueue");
959
960 return NS_OK;
961 }
962
ResolveSetCDMPromiseIfDone(TrackType aTrack)963 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {
964 // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
965 // and would create new one for specific track in the next Update.
966 MOZ_ASSERT(OnTaskQueue());
967
968 if (mSetCDMPromise.IsEmpty()) {
969 return true;
970 }
971
972 MOZ_ASSERT(mCDMProxy);
973 if (mSetCDMForTracks.contains(aTrack)) {
974 mSetCDMForTracks -= aTrack;
975 }
976
977 if (mSetCDMForTracks.isEmpty()) {
978 LOGV("%s : Done ", __func__);
979 mSetCDMPromise.Resolve(/* aIgnored = */ true, __func__);
980 if (HasAudio()) {
981 ScheduleUpdate(TrackInfo::kAudioTrack);
982 }
983 if (HasVideo()) {
984 ScheduleUpdate(TrackInfo::kVideoTrack);
985 }
986 return true;
987 }
988 LOGV("%s : %s track is ready.", __func__, TrackTypeToStr(aTrack));
989 return false;
990 }
991
PrepareToSetCDMForTrack(TrackType aTrack)992 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack) {
993 MOZ_ASSERT(OnTaskQueue());
994 LOGV("%s : %s", __func__, TrackTypeToStr(aTrack));
995
996 mSetCDMForTracks += aTrack;
997 if (mCDMProxy) {
998 // An old cdm proxy exists, so detaching old cdm proxy by shutting down
999 // MediaDataDecoder.
1000 ShutdownDecoder(aTrack);
1001 }
1002 ScheduleUpdate(aTrack);
1003 }
1004
IsDecoderWaitingForCDM(TrackType aTrack)1005 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack) {
1006 MOZ_ASSERT(OnTaskQueue());
1007 return GetDecoderData(aTrack).IsEncrypted() &&
1008 mSetCDMForTracks.contains(aTrack) && !mCDMProxy;
1009 }
1010
SetCDMProxy(CDMProxy * aProxy)1011 RefPtr<SetCDMPromise> MediaFormatReader::SetCDMProxy(CDMProxy* aProxy) {
1012 MOZ_ASSERT(OnTaskQueue());
1013 LOGV("SetCDMProxy (%p)", aProxy);
1014
1015 if (mShutdown) {
1016 return SetCDMPromise::CreateAndReject(
1017 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1018 "MediaFormatReader is shutting down"),
1019 __func__);
1020 }
1021
1022 mSetCDMPromise.RejectIfExists(
1023 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1024 "Another new CDM proxy is being set."),
1025 __func__);
1026
1027 // Shutdown all decoders as switching CDM proxy indicates that it's
1028 // inappropriate for the existing decoders to continue decoding via the old
1029 // CDM proxy.
1030 if (HasAudio()) {
1031 PrepareToSetCDMForTrack(TrackInfo::kAudioTrack);
1032 }
1033 if (HasVideo()) {
1034 PrepareToSetCDMForTrack(TrackInfo::kVideoTrack);
1035 }
1036
1037 mCDMProxy = aProxy;
1038
1039 // Release old PDMFactory which contains an EMEDecoderModule.
1040 mEncryptedPlatform = nullptr;
1041
1042 if (!mInitDone || mSetCDMForTracks.isEmpty() || !mCDMProxy) {
1043 // 1) MFR is not initialized yet or
1044 // 2) Demuxer is initialized without active audio and video or
1045 // 3) A null cdm proxy is set
1046 // the promise can be resolved directly.
1047 mSetCDMForTracks.clear();
1048 return SetCDMPromise::CreateAndResolve(/* aIgnored = */ true, __func__);
1049 }
1050
1051 RefPtr<SetCDMPromise> p = mSetCDMPromise.Ensure(__func__);
1052 return p;
1053 }
1054
IsWaitingOnCDMResource()1055 bool MediaFormatReader::IsWaitingOnCDMResource() {
1056 MOZ_ASSERT(OnTaskQueue());
1057 return IsEncrypted() && !mCDMProxy;
1058 }
1059
1060 RefPtr<MediaFormatReader::MetadataPromise>
AsyncReadMetadata()1061 MediaFormatReader::AsyncReadMetadata() {
1062 AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK);
1063 MOZ_ASSERT(OnTaskQueue());
1064
1065 MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
1066
1067 if (mInitDone) {
1068 // We are returning from dormant.
1069 MetadataHolder metadata;
1070 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1071 return MetadataPromise::CreateAndResolve(std::move(metadata), __func__);
1072 }
1073
1074 RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
1075
1076 mDemuxer->Init()
1077 ->Then(OwnerThread(), __func__, this,
1078 &MediaFormatReader::OnDemuxerInitDone,
1079 &MediaFormatReader::OnDemuxerInitFailed)
1080 ->Track(mDemuxerInitRequest);
1081 return p;
1082 }
1083
OnDemuxerInitDone(const MediaResult & aResult)1084 void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
1085 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK);
1086 MOZ_ASSERT(OnTaskQueue());
1087 mDemuxerInitRequest.Complete();
1088
1089 if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1090 mMetadataPromise.Reject(aResult, __func__);
1091 return;
1092 }
1093
1094 mDemuxerInitDone = true;
1095
1096 UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
1097
1098 RefPtr<PDMFactory> platform;
1099 if (!IsWaitingOnCDMResource()) {
1100 platform = new PDMFactory();
1101 }
1102
1103 // To decode, we need valid video and a place to put it.
1104 bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
1105 GetImageContainer();
1106
1107 if (videoActive) {
1108 // We currently only handle the first video track.
1109 MutexAutoLock lock(mVideo.mMutex);
1110 mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1111 if (!mVideo.mTrackDemuxer) {
1112 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1113 return;
1114 }
1115
1116 UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
1117 videoActive = videoInfo && videoInfo->IsValid();
1118 if (videoActive) {
1119 if (platform &&
1120 !platform->SupportsMimeType(videoInfo->mMimeType, nullptr)) {
1121 // We have no decoder for this track. Error.
1122 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1123 return;
1124 }
1125 mInfo.mVideo = *videoInfo->GetAsVideoInfo();
1126 mVideo.mWorkingInfo = MakeUnique<VideoInfo>(mInfo.mVideo);
1127 for (const MetadataTag& tag : videoInfo->mTags) {
1128 tags->Put(tag.mKey, tag.mValue);
1129 }
1130 mVideo.mOriginalInfo = std::move(videoInfo);
1131 mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
1132 } else {
1133 mVideo.mTrackDemuxer->BreakCycles();
1134 mVideo.mTrackDemuxer = nullptr;
1135 }
1136 }
1137
1138 bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1139 if (audioActive) {
1140 MutexAutoLock lock(mAudio.mMutex);
1141 mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1142 if (!mAudio.mTrackDemuxer) {
1143 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1144 return;
1145 }
1146
1147 UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
1148 // We actively ignore audio tracks that we know we can't play.
1149 audioActive = audioInfo && audioInfo->IsValid() &&
1150 (!platform ||
1151 platform->SupportsMimeType(audioInfo->mMimeType, nullptr));
1152
1153 if (audioActive) {
1154 mInfo.mAudio = *audioInfo->GetAsAudioInfo();
1155 mAudio.mWorkingInfo = MakeUnique<AudioInfo>(mInfo.mAudio);
1156 for (const MetadataTag& tag : audioInfo->mTags) {
1157 tags->Put(tag.mKey, tag.mValue);
1158 }
1159 mAudio.mOriginalInfo = std::move(audioInfo);
1160 mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
1161 } else {
1162 mAudio.mTrackDemuxer->BreakCycles();
1163 mAudio.mTrackDemuxer = nullptr;
1164 }
1165 }
1166
1167 UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
1168 if (crypto && crypto->IsEncrypted()) {
1169 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1170 for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
1171 mOnEncrypted.Notify(crypto->mInitDatas[i].mInitData,
1172 crypto->mInitDatas[i].mType);
1173 }
1174 mInfo.mCrypto = *crypto;
1175 }
1176
1177 auto videoDuration = HasVideo() ? mInfo.mVideo.mDuration : TimeUnit::Zero();
1178 auto audioDuration = HasAudio() ? mInfo.mAudio.mDuration : TimeUnit::Zero();
1179
1180 auto duration = std::max(videoDuration, audioDuration);
1181 if (duration.IsPositive()) {
1182 mInfo.mMetadataDuration = Some(duration);
1183 }
1184
1185 mInfo.mMediaSeekable = mDemuxer->IsSeekable();
1186 mInfo.mMediaSeekableOnlyInBufferedRanges =
1187 mDemuxer->IsSeekableOnlyInBufferedRanges();
1188
1189 if (!videoActive && !audioActive) {
1190 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1191 return;
1192 }
1193
1194 mTags = std::move(tags);
1195 mInitDone = true;
1196
1197 // Try to get the start time.
1198 // For MSE case, the start time of each track is assumed to be 0.
1199 // For others, we must demux the first sample to know the start time for each
1200 // track.
1201 if (!mDemuxer->ShouldComputeStartTime()) {
1202 mAudio.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1203 mVideo.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1204 } else {
1205 if (HasAudio()) {
1206 RequestDemuxSamples(TrackInfo::kAudioTrack);
1207 }
1208
1209 if (HasVideo()) {
1210 RequestDemuxSamples(TrackInfo::kVideoTrack);
1211 }
1212 }
1213
1214 if (aResult != NS_OK) {
1215 mOnDecodeWarning.Notify(aResult);
1216 }
1217
1218 MaybeResolveMetadataPromise();
1219 }
1220
MaybeResolveMetadataPromise()1221 void MediaFormatReader::MaybeResolveMetadataPromise() {
1222 MOZ_ASSERT(OnTaskQueue());
1223
1224 if ((HasAudio() && mAudio.mFirstDemuxedSampleTime.isNothing()) ||
1225 (HasVideo() && mVideo.mFirstDemuxedSampleTime.isNothing())) {
1226 return;
1227 }
1228
1229 TimeUnit startTime =
1230 std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
1231 mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
1232
1233 if (!startTime.IsInfinite()) {
1234 mInfo.mStartTime = startTime; // mInfo.mStartTime is initialized to 0.
1235 }
1236
1237 MetadataHolder metadata;
1238 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1239 metadata.mTags = mTags->Count() ? std::move(mTags) : nullptr;
1240
1241 // We now have all the informations required to calculate the initial buffered
1242 // range.
1243 mHasStartTime = true;
1244 UpdateBuffered();
1245
1246 mMetadataPromise.Resolve(std::move(metadata), __func__);
1247 }
1248
IsEncrypted() const1249 bool MediaFormatReader::IsEncrypted() const {
1250 return (HasAudio() && mAudio.GetCurrentInfo()->mCrypto.IsEncrypted()) ||
1251 (HasVideo() && mVideo.GetCurrentInfo()->mCrypto.IsEncrypted());
1252 }
1253
OnDemuxerInitFailed(const MediaResult & aError)1254 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError) {
1255 mDemuxerInitRequest.Complete();
1256 mMetadataPromise.Reject(aError, __func__);
1257 }
1258
ReadUpdatedMetadata(MediaInfo * aInfo)1259 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) {
1260 // Called on the MDSM's TaskQueue.
1261 {
1262 MutexAutoLock lock(mVideo.mMutex);
1263 if (HasVideo()) {
1264 aInfo->mVideo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1265 }
1266 }
1267 {
1268 MutexAutoLock lock(mAudio.mMutex);
1269 if (HasAudio()) {
1270 aInfo->mAudio = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1271 }
1272 }
1273 }
1274
GetDecoderData(TrackType aTrack)1275 MediaFormatReader::DecoderData& MediaFormatReader::GetDecoderData(
1276 TrackType aTrack) {
1277 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
1278 aTrack == TrackInfo::kVideoTrack);
1279 if (aTrack == TrackInfo::kAudioTrack) {
1280 return mAudio;
1281 }
1282 return mVideo;
1283 }
1284
ShouldSkip(TimeUnit aTimeThreshold)1285 bool MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold) {
1286 MOZ_ASSERT(HasVideo());
1287
1288 if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1289 return false;
1290 }
1291
1292 TimeUnit nextKeyframe;
1293 nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
1294 if (NS_FAILED(rv)) {
1295 // Only OggTrackDemuxer with video type gets into here.
1296 // We don't support skip-to-next-frame for this case.
1297 return false;
1298 }
1299 return (nextKeyframe <= aTimeThreshold ||
1300 (mVideo.mTimeThreshold &&
1301 mVideo.mTimeThreshold.ref().EndTime() < aTimeThreshold)) &&
1302 nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
1303 }
1304
RequestVideoData(const TimeUnit & aTimeThreshold)1305 RefPtr<MediaFormatReader::VideoDataPromise> MediaFormatReader::RequestVideoData(
1306 const TimeUnit& aTimeThreshold) {
1307 MOZ_ASSERT(OnTaskQueue());
1308 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1309 "No sample requests allowed while seeking");
1310 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
1311 MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
1312 mVideo.mTimeThreshold.isSome());
1313 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1314 LOGV("RequestVideoData(%" PRId64 ")", aTimeThreshold.ToMicroseconds());
1315
1316 if (!HasVideo()) {
1317 LOG("called with no video track");
1318 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1319 __func__);
1320 }
1321
1322 if (IsSeeking()) {
1323 LOG("called mid-seek. Rejecting.");
1324 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1325 __func__);
1326 }
1327
1328 if (mShutdown) {
1329 NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1330 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1331 __func__);
1332 }
1333
1334 // Ensure we have no pending seek going as ShouldSkip could return out of date
1335 // information.
1336 if (!mVideo.HasInternalSeekPending() && ShouldSkip(aTimeThreshold)) {
1337 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1338 SkipVideoDemuxToNextKeyFrame(aTimeThreshold);
1339 return p;
1340 }
1341
1342 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1343 ScheduleUpdate(TrackInfo::kVideoTrack);
1344
1345 return p;
1346 }
1347
OnDemuxFailed(TrackType aTrack,const MediaResult & aError)1348 void MediaFormatReader::OnDemuxFailed(TrackType aTrack,
1349 const MediaResult& aError) {
1350 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK);
1351 MOZ_ASSERT(OnTaskQueue());
1352 LOG("Failed to demux %s, failure:%s",
1353 aTrack == TrackType::kVideoTrack ? "video" : "audio",
1354 aError.ErrorName().get());
1355 auto& decoder = GetDecoderData(aTrack);
1356 decoder.mDemuxRequest.Complete();
1357 switch (aError.Code()) {
1358 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1359 DDLOG(DDLogCategory::Log,
1360 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1361 : "audio_demux_interruption",
1362 aError);
1363 if (!decoder.mWaitingForData) {
1364 decoder.RequestDrain();
1365 }
1366 NotifyEndOfStream(aTrack);
1367 break;
1368 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1369 DDLOG(DDLogCategory::Log,
1370 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1371 : "audio_demux_interruption",
1372 aError);
1373 if (!decoder.mWaitingForData) {
1374 decoder.RequestDrain();
1375 }
1376 NotifyWaitingForData(aTrack);
1377 break;
1378 case NS_ERROR_DOM_MEDIA_CANCELED:
1379 DDLOG(DDLogCategory::Log,
1380 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1381 : "audio_demux_interruption",
1382 aError);
1383 if (decoder.HasPromise()) {
1384 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
1385 }
1386 break;
1387 default:
1388 DDLOG(DDLogCategory::Log,
1389 aTrack == TrackType::kVideoTrack ? "video_demux_error"
1390 : "audio_demux_error",
1391 aError);
1392 NotifyError(aTrack, aError);
1393 break;
1394 }
1395 }
1396
DoDemuxVideo()1397 void MediaFormatReader::DoDemuxVideo() {
1398 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK);
1399 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1400
1401 DDLOG(DDLogCategory::Log, "video_demuxing", DDNoValue{});
1402 auto p = mVideo.mTrackDemuxer->GetSamples(1);
1403
1404 if (mVideo.mFirstDemuxedSampleTime.isNothing()) {
1405 RefPtr<MediaFormatReader> self = this;
1406 p = p->Then(
1407 OwnerThread(), __func__,
1408 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1409 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1410 MEDIA_PLAYBACK);
1411 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxed",
1412 DDNoValue{});
1413 self->OnFirstDemuxCompleted(TrackInfo::kVideoTrack, aSamples);
1414 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1415 },
1416 [self](const MediaResult& aError) {
1417 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1418 MEDIA_PLAYBACK);
1419 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxing_error",
1420 aError);
1421 self->OnFirstDemuxFailed(TrackInfo::kVideoTrack, aError);
1422 return SamplesPromise::CreateAndReject(aError, __func__);
1423 });
1424 }
1425
1426 p->Then(OwnerThread(), __func__, this,
1427 &MediaFormatReader::OnVideoDemuxCompleted,
1428 &MediaFormatReader::OnVideoDemuxFailed)
1429 ->Track(mVideo.mDemuxRequest);
1430 }
1431
OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)1432 void MediaFormatReader::OnVideoDemuxCompleted(
1433 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1434 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1435 MEDIA_PLAYBACK);
1436 LOGV("%zu video samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1437 aSamples->GetSamples()[0]->mTrackInfo
1438 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1439 : 0);
1440 DDLOG(DDLogCategory::Log, "video_demuxed_samples",
1441 uint64_t(aSamples->GetSamples().Length()));
1442 mVideo.mDemuxRequest.Complete();
1443 mVideo.mQueuedSamples.AppendElements(aSamples->GetSamples());
1444 ScheduleUpdate(TrackInfo::kVideoTrack);
1445 }
1446
1447 RefPtr<MediaFormatReader::AudioDataPromise>
RequestAudioData()1448 MediaFormatReader::RequestAudioData() {
1449 MOZ_ASSERT(OnTaskQueue());
1450 MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
1451 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || mSeekPromise.IsEmpty(),
1452 "No sample requests allowed while seeking");
1453 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !mAudio.mSeekRequest.Exists() ||
1454 mAudio.mTimeThreshold.isSome());
1455 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !IsSeeking(), "called mid-seek");
1456 LOGV("");
1457
1458 if (!HasAudio()) {
1459 LOG("called with no audio track");
1460 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1461 __func__);
1462 }
1463
1464 if (IsSeeking()) {
1465 LOG("called mid-seek. Rejecting.");
1466 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1467 __func__);
1468 }
1469
1470 if (mShutdown) {
1471 NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1472 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1473 __func__);
1474 }
1475
1476 RefPtr<AudioDataPromise> p = mAudio.EnsurePromise(__func__);
1477 ScheduleUpdate(TrackInfo::kAudioTrack);
1478
1479 return p;
1480 }
1481
DoDemuxAudio()1482 void MediaFormatReader::DoDemuxAudio() {
1483 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK);
1484 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1485
1486 DDLOG(DDLogCategory::Log, "audio_demuxing", DDNoValue{});
1487 auto p = mAudio.mTrackDemuxer->GetSamples(1);
1488
1489 if (mAudio.mFirstDemuxedSampleTime.isNothing()) {
1490 RefPtr<MediaFormatReader> self = this;
1491 p = p->Then(
1492 OwnerThread(), __func__,
1493 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1494 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1495 MEDIA_PLAYBACK);
1496 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxed",
1497 DDNoValue{});
1498 self->OnFirstDemuxCompleted(TrackInfo::kAudioTrack, aSamples);
1499 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1500 },
1501 [self](const MediaResult& aError) {
1502 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1503 MEDIA_PLAYBACK);
1504 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxing_error",
1505 aError);
1506 self->OnFirstDemuxFailed(TrackInfo::kAudioTrack, aError);
1507 return SamplesPromise::CreateAndReject(aError, __func__);
1508 });
1509 }
1510
1511 p->Then(OwnerThread(), __func__, this,
1512 &MediaFormatReader::OnAudioDemuxCompleted,
1513 &MediaFormatReader::OnAudioDemuxFailed)
1514 ->Track(mAudio.mDemuxRequest);
1515 }
1516
OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)1517 void MediaFormatReader::OnAudioDemuxCompleted(
1518 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1519 LOGV("%zu audio samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1520 aSamples->GetSamples()[0]->mTrackInfo
1521 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1522 : 0);
1523 DDLOG(DDLogCategory::Log, "audio_demuxed_samples",
1524 uint64_t(aSamples->GetSamples().Length()));
1525 mAudio.mDemuxRequest.Complete();
1526 mAudio.mQueuedSamples.AppendElements(aSamples->GetSamples());
1527 ScheduleUpdate(TrackInfo::kAudioTrack);
1528 }
1529
NotifyNewOutput(TrackType aTrack,MediaDataDecoder::DecodedData && aResults)1530 void MediaFormatReader::NotifyNewOutput(
1531 TrackType aTrack, MediaDataDecoder::DecodedData&& aResults) {
1532 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK);
1533 MOZ_ASSERT(OnTaskQueue());
1534 auto& decoder = GetDecoderData(aTrack);
1535 if (aResults.IsEmpty()) {
1536 DDLOG(DDLogCategory::Log,
1537 aTrack == TrackInfo::kAudioTrack ? "decoded_audio" : "decoded_video",
1538 "no output samples");
1539 } else
1540 for (auto&& sample : aResults) {
1541 if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1542 switch (sample->mType) {
1543 case MediaData::Type::AUDIO_DATA:
1544 DDLOGPR(DDLogCategory::Log,
1545 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1546 : "decoded_got_audio!?",
1547 "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1548 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1549 ", \"duration_us\":%" PRIi64 ", \"frames\":%" PRIu32
1550 ", \"channels\":%" PRIu32 ", \"rate\":%" PRIu32
1551 ", \"bytes\":%zu}",
1552 sample->mOffset, sample->mTime.ToMicroseconds(),
1553 sample->mTimecode.ToMicroseconds(),
1554 sample->mDuration.ToMicroseconds(),
1555 sample->As<AudioData>()->Frames(),
1556 sample->As<AudioData>()->mChannels,
1557 sample->As<AudioData>()->mRate,
1558 sample->As<AudioData>()->Data().Length());
1559 break;
1560 case MediaData::Type::VIDEO_DATA:
1561 DDLOGPR(DDLogCategory::Log,
1562 aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1563 : "decoded_got_video!?",
1564 "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1565 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1566 ", \"duration_us\":%" PRIi64
1567 ", \"kf\":%s, \"size\":[%" PRIi32 ",%" PRIi32 "]}",
1568 sample->mOffset, sample->mTime.ToMicroseconds(),
1569 sample->mTimecode.ToMicroseconds(),
1570 sample->mDuration.ToMicroseconds(),
1571 sample->mKeyframe ? "true" : "false",
1572 sample->As<VideoData>()->mDisplay.width,
1573 sample->As<VideoData>()->mDisplay.height);
1574 break;
1575 case MediaData::Type::RAW_DATA:
1576 DDLOGPR(DDLogCategory::Log,
1577 aTrack == TrackInfo::kAudioTrack
1578 ? "decoded_audio"
1579 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1580 : "decoded_?",
1581 "{\"type\":\"RawData\", \"offset\":%" PRIi64
1582 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1583 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1584 sample->mOffset, sample->mTime.ToMicroseconds(),
1585 sample->mTimecode.ToMicroseconds(),
1586 sample->mDuration.ToMicroseconds(),
1587 sample->mKeyframe ? "true" : "false");
1588 break;
1589 case MediaData::Type::NULL_DATA:
1590 DDLOGPR(DDLogCategory::Log,
1591 aTrack == TrackInfo::kAudioTrack
1592 ? "decoded_audio"
1593 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1594 : "decoded_?",
1595 "{\"type\":\"NullData\", \"offset\":%" PRIi64
1596 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1597 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1598 sample->mOffset, sample->mTime.ToMicroseconds(),
1599 sample->mTimecode.ToMicroseconds(),
1600 sample->mDuration.ToMicroseconds(),
1601 sample->mKeyframe ? "true" : "false");
1602 break;
1603 }
1604 }
1605 LOGV("Received new %s sample time:%" PRId64 " duration:%" PRId64,
1606 TrackTypeToStr(aTrack), sample->mTime.ToMicroseconds(),
1607 sample->mDuration.ToMicroseconds());
1608 decoder.mOutput.AppendElement(sample);
1609 decoder.mNumSamplesOutput++;
1610 decoder.mNumOfConsecutiveError = 0;
1611 }
1612 LOG("Done processing new %s samples", TrackTypeToStr(aTrack));
1613
1614 if (!aResults.IsEmpty()) {
1615 // We have decoded our first frame, we can now starts to skip future errors.
1616 decoder.mFirstFrameTime.reset();
1617 }
1618 ScheduleUpdate(aTrack);
1619 }
1620
NotifyError(TrackType aTrack,const MediaResult & aError)1621 void MediaFormatReader::NotifyError(TrackType aTrack,
1622 const MediaResult& aError) {
1623 MOZ_ASSERT(OnTaskQueue());
1624 NS_WARNING(aError.Description().get());
1625 LOGV("%s Decoding error", TrackTypeToStr(aTrack));
1626 auto& decoder = GetDecoderData(aTrack);
1627 decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
1628
1629 ScheduleUpdate(aTrack);
1630 }
1631
NotifyWaitingForData(TrackType aTrack)1632 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack) {
1633 MOZ_ASSERT(OnTaskQueue());
1634 auto& decoder = GetDecoderData(aTrack);
1635 decoder.mWaitingForData = true;
1636 if (decoder.mTimeThreshold) {
1637 decoder.mTimeThreshold.ref().mWaiting = true;
1638 }
1639 ScheduleUpdate(aTrack);
1640 }
1641
NotifyWaitingForKey(TrackType aTrack)1642 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack) {
1643 MOZ_ASSERT(OnTaskQueue());
1644 auto& decoder = GetDecoderData(aTrack);
1645 mOnWaitingForKey.Notify();
1646 if (!decoder.mDecodeRequest.Exists()) {
1647 LOGV("WaitingForKey received while no pending decode. Ignoring");
1648 return;
1649 }
1650 decoder.mWaitingForKey = true;
1651 ScheduleUpdate(aTrack);
1652 }
1653
NotifyEndOfStream(TrackType aTrack)1654 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack) {
1655 MOZ_ASSERT(OnTaskQueue());
1656 auto& decoder = GetDecoderData(aTrack);
1657 decoder.mDemuxEOS = true;
1658 ScheduleUpdate(aTrack);
1659 }
1660
NeedInput(DecoderData & aDecoder)1661 bool MediaFormatReader::NeedInput(DecoderData& aDecoder) {
1662 // The decoder will not be fed a new raw sample until the current decoding
1663 // requests has completed.
1664 return (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
1665 !aDecoder.HasPendingDrain() && !aDecoder.HasFatalError() &&
1666 !aDecoder.mDemuxRequest.Exists() && !aDecoder.mOutput.Length() &&
1667 !aDecoder.HasInternalSeekPending() &&
1668 !aDecoder.mDecodeRequest.Exists();
1669 }
1670
ScheduleUpdate(TrackType aTrack)1671 void MediaFormatReader::ScheduleUpdate(TrackType aTrack) {
1672 MOZ_ASSERT(OnTaskQueue());
1673 if (mShutdown) {
1674 return;
1675 }
1676 auto& decoder = GetDecoderData(aTrack);
1677 MOZ_RELEASE_ASSERT(decoder.GetCurrentInfo(),
1678 "Can only schedule update when track exists");
1679
1680 if (decoder.mUpdateScheduled) {
1681 return;
1682 }
1683 LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
1684 decoder.mUpdateScheduled = true;
1685 RefPtr<nsIRunnable> task(NewRunnableMethod<TrackType>(
1686 "MediaFormatReader::Update", this, &MediaFormatReader::Update, aTrack));
1687 nsresult rv = OwnerThread()->Dispatch(task.forget());
1688 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1689 Unused << rv;
1690 }
1691
UpdateReceivedNewData(TrackType aTrack)1692 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) {
1693 MOZ_ASSERT(OnTaskQueue());
1694 auto& decoder = GetDecoderData(aTrack);
1695
1696 if (!decoder.mReceivedNewData) {
1697 return false;
1698 }
1699
1700 // We do not want to clear mWaitingForData while there are pending
1701 // demuxing or seeking operations that could affect the value of this flag.
1702 // This is in order to ensure that we will retry once they complete as we may
1703 // now have new data that could potentially allow those operations to
1704 // successfully complete if tried again.
1705 if (decoder.mSeekRequest.Exists()) {
1706 // Nothing more to do until this operation complete.
1707 return true;
1708 }
1709
1710 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
1711 LOGV("Skipping in progress, nothing more to do");
1712 return true;
1713 }
1714
1715 if (decoder.mDemuxRequest.Exists()) {
1716 // We may have pending operations to process, so we want to continue
1717 // after UpdateReceivedNewData returns.
1718 return false;
1719 }
1720
1721 if (decoder.HasPendingDrain()) {
1722 // We do not want to clear mWaitingForData or mDemuxEOS while
1723 // a drain is in progress in order to properly complete the operation.
1724 return false;
1725 }
1726
1727 decoder.mReceivedNewData = false;
1728 if (decoder.mTimeThreshold) {
1729 decoder.mTimeThreshold.ref().mWaiting = false;
1730 }
1731 decoder.mWaitingForData = false;
1732
1733 if (decoder.HasFatalError()) {
1734 return false;
1735 }
1736
1737 if (!mSeekPromise.IsEmpty() &&
1738 (!IsVideoSeeking() || aTrack == TrackInfo::kVideoTrack)) {
1739 MOZ_ASSERT(!decoder.HasPromise());
1740 MOZ_DIAGNOSTIC_ASSERT(
1741 (IsVideoSeeking() || !mAudio.mTimeThreshold) && !mVideo.mTimeThreshold,
1742 "InternalSeek must have been aborted when Seek was first called");
1743 MOZ_DIAGNOSTIC_ASSERT(
1744 (IsVideoSeeking() || !mAudio.HasWaitingPromise()) &&
1745 !mVideo.HasWaitingPromise(),
1746 "Waiting promises must have been rejected when Seek was first called");
1747 if (mVideo.mSeekRequest.Exists() ||
1748 (!IsVideoSeeking() && mAudio.mSeekRequest.Exists())) {
1749 // Already waiting for a seek to complete. Nothing more to do.
1750 return true;
1751 }
1752 LOG("Attempting Seek");
1753 ScheduleSeek();
1754 return true;
1755 }
1756 if (decoder.HasInternalSeekPending() || decoder.HasWaitingPromise()) {
1757 if (decoder.HasInternalSeekPending()) {
1758 LOG("Attempting Internal Seek");
1759 InternalSeek(aTrack, decoder.mTimeThreshold.ref());
1760 }
1761 if (decoder.HasWaitingPromise() && !decoder.IsWaitingForKey() &&
1762 !decoder.IsWaitingForData()) {
1763 MOZ_ASSERT(!decoder.HasPromise());
1764 LOG("We have new data. Resolving WaitingPromise");
1765 decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
1766 }
1767 return true;
1768 }
1769 return false;
1770 }
1771
RequestDemuxSamples(TrackType aTrack)1772 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack) {
1773 MOZ_ASSERT(OnTaskQueue());
1774 auto& decoder = GetDecoderData(aTrack);
1775 MOZ_ASSERT(!decoder.mDemuxRequest.Exists());
1776
1777 if (!decoder.mQueuedSamples.IsEmpty()) {
1778 // No need to demux new samples.
1779 return;
1780 }
1781
1782 if (decoder.mDemuxEOS) {
1783 // Nothing left to demux.
1784 // We do not want to attempt to demux while in waiting for data mode
1785 // as it would retrigger an unnecessary drain.
1786 return;
1787 }
1788
1789 LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
1790 if (aTrack == TrackInfo::kVideoTrack) {
1791 DoDemuxVideo();
1792 } else {
1793 DoDemuxAudio();
1794 }
1795 }
1796
DecodeDemuxedSamples(TrackType aTrack,MediaRawData * aSample)1797 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
1798 MediaRawData* aSample) {
1799 MOZ_ASSERT(OnTaskQueue());
1800 auto& decoder = GetDecoderData(aTrack);
1801 RefPtr<MediaFormatReader> self = this;
1802 decoder.mFlushed = false;
1803 DDLOGPR(DDLogCategory::Log,
1804 aTrack == TrackInfo::kAudioTrack
1805 ? "decode_audio"
1806 : aTrack == TrackInfo::kVideoTrack ? "decode_video" : "decode_?",
1807 "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1808 ", \"bytes\":%zu, \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1809 ", \"duration_us\":%" PRIi64 ",%s%s}",
1810 aSample->mOffset, aSample->Size(), aSample->mTime.ToMicroseconds(),
1811 aSample->mTimecode.ToMicroseconds(),
1812 aSample->mDuration.ToMicroseconds(), aSample->mKeyframe ? " kf" : "",
1813 aSample->mEOS ? " eos" : "");
1814 decoder.mDecoder->Decode(aSample)
1815 ->Then(
1816 mTaskQueue, __func__,
1817 [self, aTrack, &decoder](MediaDataDecoder::DecodedData&& aResults) {
1818 decoder.mDecodeRequest.Complete();
1819 self->NotifyNewOutput(aTrack, std::move(aResults));
1820 },
1821 [self, aTrack, &decoder](const MediaResult& aError) {
1822 decoder.mDecodeRequest.Complete();
1823 self->NotifyError(aTrack, aError);
1824 })
1825 ->Track(decoder.mDecodeRequest);
1826 }
1827
HandleDemuxedSamples(TrackType aTrack,FrameStatistics::AutoNotifyDecoded & aA)1828 void MediaFormatReader::HandleDemuxedSamples(
1829 TrackType aTrack, FrameStatistics::AutoNotifyDecoded& aA) {
1830 MOZ_ASSERT(OnTaskQueue());
1831
1832 auto& decoder = GetDecoderData(aTrack);
1833
1834 if (decoder.mFlushing) {
1835 LOGV("Decoder operation in progress, let it complete.");
1836 return;
1837 }
1838
1839 if (decoder.mQueuedSamples.IsEmpty()) {
1840 return;
1841 }
1842
1843 RefPtr<MediaRawData> sample = decoder.mQueuedSamples[0];
1844 const RefPtr<TrackInfoSharedPtr> info = sample->mTrackInfo;
1845
1846 if (info && decoder.mLastStreamSourceID != info->GetID()) {
1847 nsTArray<RefPtr<MediaRawData>> samples;
1848 if (decoder.mDecoder) {
1849 bool recyclable =
1850 StaticPrefs::media_decoder_recycle_enabled() &&
1851 decoder.mDecoder->SupportDecoderRecycling() &&
1852 (*info)->mCrypto.mCryptoScheme ==
1853 decoder.GetCurrentInfo()->mCrypto.mCryptoScheme &&
1854 (*info)->mMimeType == decoder.GetCurrentInfo()->mMimeType;
1855 if (!recyclable && decoder.mTimeThreshold.isNothing() &&
1856 (decoder.mNextStreamSourceID.isNothing() ||
1857 decoder.mNextStreamSourceID.ref() != info->GetID())) {
1858 LOG("%s stream id has changed from:%d to:%d, draining decoder.",
1859 TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
1860 decoder.RequestDrain();
1861 decoder.mNextStreamSourceID = Some(info->GetID());
1862 ScheduleUpdate(aTrack);
1863 return;
1864 }
1865
1866 // If flushing is required, it will clear our array of queued samples.
1867 // So we may need to make a copy.
1868 samples = decoder.mQueuedSamples.Clone();
1869 if (!recyclable) {
1870 LOG("Decoder does not support recycling, recreate decoder.");
1871 ShutdownDecoder(aTrack);
1872 // We're going to be using a new decoder following the change of content
1873 // We can attempt to use hardware decoding again.
1874 decoder.mHardwareDecodingDisabled = false;
1875 } else if (decoder.HasWaitingPromise()) {
1876 decoder.Flush();
1877 }
1878 }
1879
1880 LOG("%s stream id has changed from:%d to:%d.", TrackTypeToStr(aTrack),
1881 decoder.mLastStreamSourceID, info->GetID());
1882
1883 if (aTrack == TrackInfo::kVideoTrack) {
1884 // We are about to create a new decoder thus the benchmark,
1885 // up to this point, is stored.
1886 NotifyDecoderBenchmarkStore();
1887 }
1888 decoder.mNextStreamSourceID.reset();
1889 decoder.mLastStreamSourceID = info->GetID();
1890 decoder.mInfo = info;
1891 {
1892 MutexAutoLock lock(decoder.mMutex);
1893 if (aTrack == TrackInfo::kAudioTrack) {
1894 decoder.mWorkingInfo = MakeUnique<AudioInfo>(*info->GetAsAudioInfo());
1895 } else if (aTrack == TrackInfo::kVideoTrack) {
1896 decoder.mWorkingInfo = MakeUnique<VideoInfo>(*info->GetAsVideoInfo());
1897 }
1898 }
1899
1900 decoder.mMeanRate.Reset();
1901
1902 if (sample->mKeyframe) {
1903 if (samples.Length()) {
1904 decoder.mQueuedSamples = std::move(samples);
1905 }
1906 } else {
1907 auto time = TimeInterval(sample->mTime, sample->GetEndTime());
1908 InternalSeekTarget seekTarget =
1909 decoder.mTimeThreshold.refOr(InternalSeekTarget(time, false));
1910 LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64,
1911 sample->mTime.ToMicroseconds());
1912 InternalSeek(aTrack, seekTarget);
1913 return;
1914 }
1915 }
1916
1917 // Calculate the average frame rate. The first frame will be accounted
1918 // for twice.
1919 decoder.mMeanRate.Update(sample->mDuration);
1920
1921 if (!decoder.mDecoder) {
1922 mDecoderFactory->CreateDecoder(aTrack);
1923 return;
1924 }
1925
1926 LOGV("Input:%" PRId64 " (dts:%" PRId64 " kf:%d)",
1927 sample->mTime.ToMicroseconds(), sample->mTimecode.ToMicroseconds(),
1928 sample->mKeyframe);
1929 decoder.mNumSamplesInput++;
1930 decoder.mSizeOfQueue++;
1931 if (aTrack == TrackInfo::kVideoTrack) {
1932 aA.mStats.mParsedFrames++;
1933 }
1934
1935 DecodeDemuxedSamples(aTrack, sample);
1936
1937 decoder.mQueuedSamples.RemoveElementAt(0);
1938 }
1939
InternalSeek(TrackType aTrack,const InternalSeekTarget & aTarget)1940 void MediaFormatReader::InternalSeek(TrackType aTrack,
1941 const InternalSeekTarget& aTarget) {
1942 MOZ_ASSERT(OnTaskQueue());
1943 LOG("%s internal seek to %f", TrackTypeToStr(aTrack),
1944 aTarget.Time().ToSeconds());
1945
1946 auto& decoder = GetDecoderData(aTrack);
1947 decoder.Flush();
1948 decoder.ResetDemuxer();
1949 decoder.mTimeThreshold = Some(aTarget);
1950 DDLOG(DDLogCategory::Log, "seeking", DDNoValue{});
1951 RefPtr<MediaFormatReader> self = this;
1952 decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().Time())
1953 ->Then(
1954 OwnerThread(), __func__,
1955 [self, aTrack](TimeUnit aTime) {
1956 DDLOGEX(self.get(), DDLogCategory::Log, "seeked", DDNoValue{});
1957 auto& decoder = self->GetDecoderData(aTrack);
1958 decoder.mSeekRequest.Complete();
1959 MOZ_ASSERT(decoder.mTimeThreshold,
1960 "Seek promise must be disconnected when "
1961 "timethreshold is reset");
1962 decoder.mTimeThreshold.ref().mHasSeeked = true;
1963 self->SetVideoDecodeThreshold();
1964 self->ScheduleUpdate(aTrack);
1965 },
1966 [self, aTrack](const MediaResult& aError) {
1967 auto& decoder = self->GetDecoderData(aTrack);
1968 decoder.mSeekRequest.Complete();
1969 switch (aError.Code()) {
1970 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1971 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
1972 aError);
1973 self->NotifyWaitingForData(aTrack);
1974 break;
1975 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1976 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
1977 aError);
1978 decoder.mTimeThreshold.reset();
1979 self->NotifyEndOfStream(aTrack);
1980 break;
1981 case NS_ERROR_DOM_MEDIA_CANCELED:
1982 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
1983 aError);
1984 decoder.mTimeThreshold.reset();
1985 break;
1986 default:
1987 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_error",
1988 aError);
1989 decoder.mTimeThreshold.reset();
1990 self->NotifyError(aTrack, aError);
1991 break;
1992 }
1993 })
1994 ->Track(decoder.mSeekRequest);
1995 }
1996
DrainDecoder(TrackType aTrack)1997 void MediaFormatReader::DrainDecoder(TrackType aTrack) {
1998 AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK);
1999 MOZ_ASSERT(OnTaskQueue());
2000
2001 auto& decoder = GetDecoderData(aTrack);
2002 if (decoder.mDrainState == DrainState::Draining) {
2003 return;
2004 }
2005 if (!decoder.mDecoder ||
2006 (decoder.mDrainState != DrainState::PartialDrainPending &&
2007 decoder.mNumSamplesInput == decoder.mNumSamplesOutput)) {
2008 // No frames to drain.
2009 LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack));
2010 decoder.mDrainState = DrainState::DrainAborted;
2011 ScheduleUpdate(aTrack);
2012 return;
2013 }
2014
2015 decoder.mDrainState = DrainState::Draining;
2016
2017 DDLOG(DDLogCategory::Log, "draining", DDNoValue{});
2018 RefPtr<MediaFormatReader> self = this;
2019 decoder.mDecoder->Drain()
2020 ->Then(
2021 mTaskQueue, __func__,
2022 [self, aTrack, &decoder](MediaDataDecoder::DecodedData&& aResults) {
2023 decoder.mDrainRequest.Complete();
2024 DDLOGEX(self.get(), DDLogCategory::Log, "drained", DDNoValue{});
2025 if (aResults.IsEmpty()) {
2026 decoder.mDrainState = DrainState::DrainCompleted;
2027 } else {
2028 self->NotifyNewOutput(aTrack, std::move(aResults));
2029 // Let's see if we have any more data available to drain.
2030 decoder.mDrainState = DrainState::PartialDrainPending;
2031 }
2032 self->ScheduleUpdate(aTrack);
2033 },
2034 [self, aTrack, &decoder](const MediaResult& aError) {
2035 decoder.mDrainRequest.Complete();
2036 DDLOGEX(self.get(), DDLogCategory::Log, "draining_error", aError);
2037 self->NotifyError(aTrack, aError);
2038 })
2039 ->Track(decoder.mDrainRequest);
2040 LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack));
2041 }
2042
Update(TrackType aTrack)2043 void MediaFormatReader::Update(TrackType aTrack) {
2044 AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK);
2045 MOZ_ASSERT(OnTaskQueue());
2046
2047 if (mShutdown) {
2048 return;
2049 }
2050
2051 LOGV("Processing update for %s", TrackTypeToStr(aTrack));
2052
2053 bool needOutput = false;
2054 auto& decoder = GetDecoderData(aTrack);
2055 decoder.mUpdateScheduled = false;
2056
2057 if (!mInitDone) {
2058 return;
2059 }
2060
2061 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
2062 LOGV("Skipping in progress, nothing more to do");
2063 return;
2064 }
2065
2066 if (UpdateReceivedNewData(aTrack)) {
2067 LOGV("Nothing more to do");
2068 return;
2069 }
2070
2071 if (decoder.mSeekRequest.Exists()) {
2072 LOGV("Seeking hasn't completed, nothing more to do");
2073 return;
2074 }
2075
2076 MOZ_DIAGNOSTIC_ASSERT(
2077 !decoder.HasInternalSeekPending() ||
2078 (!decoder.mOutput.Length() && !decoder.mQueuedSamples.Length()),
2079 "No frames can be demuxed or decoded while an internal seek is pending");
2080
2081 // Record number of frames decoded and parsed. Automatically update the
2082 // stats counters using the AutoNotifyDecoded stack-based class.
2083 FrameStatistics::AutoNotifyDecoded a(mFrameStats);
2084
2085 // Drop any frames found prior our internal seek target.
2086 while (decoder.mTimeThreshold && decoder.mOutput.Length()) {
2087 RefPtr<MediaData>& output = decoder.mOutput[0];
2088 InternalSeekTarget target = decoder.mTimeThreshold.ref();
2089 auto time = output->mTime;
2090 if (time >= target.Time()) {
2091 // We have reached our internal seek target.
2092 decoder.mTimeThreshold.reset();
2093 // We might have dropped some keyframes.
2094 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2095 }
2096 if (time < target.Time() || (target.mDropTarget && target.Contains(time))) {
2097 LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2098 TrackTypeToStr(aTrack), output->mTime.ToSeconds(),
2099 target.Time().ToSeconds(), output->mKeyframe);
2100 decoder.mOutput.RemoveElementAt(0);
2101 decoder.mSizeOfQueue -= 1;
2102 }
2103 }
2104
2105 while (decoder.mOutput.Length() &&
2106 decoder.mOutput[0]->mType == MediaData::Type::NULL_DATA) {
2107 LOGV("Dropping null data. Time: %" PRId64,
2108 decoder.mOutput[0]->mTime.ToMicroseconds());
2109 decoder.mOutput.RemoveElementAt(0);
2110 decoder.mSizeOfQueue -= 1;
2111 }
2112
2113 if (decoder.HasPromise()) {
2114 needOutput = true;
2115 if (decoder.mOutput.Length()) {
2116 RefPtr<MediaData> output = decoder.mOutput[0];
2117 decoder.mOutput.RemoveElementAt(0);
2118 decoder.mSizeOfQueue -= 1;
2119 decoder.mLastDecodedSampleTime =
2120 Some(TimeInterval(output->mTime, output->GetEndTime()));
2121 decoder.mNumSamplesOutputTotal++;
2122 ReturnOutput(output, aTrack);
2123 // We have a decoded sample ready to be returned.
2124 if (aTrack == TrackType::kVideoTrack) {
2125 uint64_t delta =
2126 decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
2127 a.mStats.mDecodedFrames = static_cast<uint32_t>(delta);
2128 mLastReportedNumDecodedFrames = decoder.mNumSamplesOutputTotal;
2129 if (output->mKeyframe) {
2130 if (mPreviousDecodedKeyframeTime_us <
2131 output->mTime.ToMicroseconds()) {
2132 // There is a previous keyframe -> Record inter-keyframe stats.
2133 uint64_t segment_us = output->mTime.ToMicroseconds() -
2134 mPreviousDecodedKeyframeTime_us;
2135 a.mStats.mInterKeyframeSum_us += segment_us;
2136 a.mStats.mInterKeyframeCount += 1;
2137 if (a.mStats.mInterKeyFrameMax_us < segment_us) {
2138 a.mStats.mInterKeyFrameMax_us = segment_us;
2139 }
2140 }
2141 mPreviousDecodedKeyframeTime_us = output->mTime.ToMicroseconds();
2142 }
2143 nsCString error;
2144 mVideo.mIsHardwareAccelerated =
2145 mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
2146 #ifdef XP_WIN
2147 // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2148 // of GPU RAM used.
2149 VideoData* videoData = output->As<VideoData>();
2150 mVideo.mIsHardwareAccelerated =
2151 mVideo.mIsHardwareAccelerated ||
2152 (videoData->mImage &&
2153 videoData->mImage->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE);
2154 #endif
2155 }
2156 } else if (decoder.HasFatalError()) {
2157 LOG("Rejecting %s promise: DECODE_ERROR", TrackTypeToStr(aTrack));
2158 decoder.RejectPromise(decoder.mError.ref(), __func__);
2159 return;
2160 } else if (decoder.HasCompletedDrain()) {
2161 if (decoder.mDemuxEOS) {
2162 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2163 if (aTrack == TrackInfo::kVideoTrack) {
2164 // End of video, store the benchmark of the decoder.
2165 NotifyDecoderBenchmarkStore();
2166 }
2167 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2168 } else if (decoder.mWaitingForData) {
2169 if (decoder.mDrainState == DrainState::DrainCompleted &&
2170 decoder.mLastDecodedSampleTime && !decoder.mNextStreamSourceID) {
2171 // We have completed draining the decoder following WaitingForData.
2172 // Set up the internal seek machinery to be able to resume from the
2173 // last sample decoded.
2174 LOG("Seeking to last sample time: %" PRId64,
2175 decoder.mLastDecodedSampleTime.ref().mStart.ToMicroseconds());
2176 InternalSeek(aTrack, InternalSeekTarget(
2177 decoder.mLastDecodedSampleTime.ref(), true));
2178 }
2179 if (!decoder.mReceivedNewData) {
2180 LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack));
2181 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2182 }
2183 }
2184
2185 decoder.mDrainState = DrainState::None;
2186
2187 // Now that draining has completed, we check if we have received
2188 // new data again as the result may now be different from the earlier
2189 // run.
2190 if (UpdateReceivedNewData(aTrack) || decoder.mSeekRequest.Exists()) {
2191 LOGV("Nothing more to do");
2192 return;
2193 }
2194 } else if (decoder.mDemuxEOS && !decoder.HasPendingDrain() &&
2195 decoder.mQueuedSamples.IsEmpty()) {
2196 // It is possible to transition from WAITING_FOR_DATA directly to EOS
2197 // state during the internal seek; in which case no draining would occur.
2198 // There is no more samples left to be decoded and we are already in
2199 // EOS state. We can immediately reject the data promise.
2200 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2201 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2202 } else if (decoder.mWaitingForKey) {
2203 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2204 TrackTypeToStr(aTrack));
2205 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2206 } else if (IsDecoderWaitingForCDM(aTrack)) {
2207 // Rejecting the promise could lead to entering buffering state for MDSM,
2208 // once a qualified(with the same key system and sessions created by the
2209 // same InitData) new cdm proxy is set, decoding can be resumed.
2210 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2211 TrackTypeToStr(aTrack));
2212 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2213 }
2214 }
2215
2216 if (decoder.mDrainState == DrainState::DrainRequested ||
2217 decoder.mDrainState == DrainState::PartialDrainPending) {
2218 if (decoder.mOutput.IsEmpty()) {
2219 DrainDecoder(aTrack);
2220 }
2221 return;
2222 }
2223
2224 if (decoder.mError && !decoder.HasFatalError()) {
2225 MOZ_RELEASE_ASSERT(!decoder.HasInternalSeekPending(),
2226 "No error can occur while an internal seek is pending");
2227
2228 nsCString error;
2229 bool firstFrameDecodingFailedWithHardware =
2230 decoder.mFirstFrameTime &&
2231 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2232 decoder.mDecoder && decoder.mDecoder->IsHardwareAccelerated(error) &&
2233 !decoder.mHardwareDecodingDisabled;
2234 bool needsNewDecoder =
2235 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER ||
2236 firstFrameDecodingFailedWithHardware;
2237 if (!needsNewDecoder &&
2238 ++decoder.mNumOfConsecutiveError > decoder.mMaxConsecutiveError) {
2239 DDLOG(DDLogCategory::Log, "too_many_decode_errors", decoder.mError.ref());
2240 NotifyError(aTrack, decoder.mError.ref());
2241 return;
2242 }
2243 if (firstFrameDecodingFailedWithHardware) {
2244 decoder.mHardwareDecodingDisabled = true;
2245 }
2246 decoder.mError.reset();
2247
2248 LOG("%s decoded error count %d", TrackTypeToStr(aTrack),
2249 decoder.mNumOfConsecutiveError);
2250
2251 if (needsNewDecoder) {
2252 LOG("Error: Need new decoder");
2253 ShutdownDecoder(aTrack);
2254 }
2255 if (decoder.mFirstFrameTime) {
2256 TimeInterval seekInterval = TimeInterval(decoder.mFirstFrameTime.ref(),
2257 decoder.mFirstFrameTime.ref());
2258 InternalSeek(aTrack, InternalSeekTarget(seekInterval, false));
2259 return;
2260 }
2261
2262 TimeUnit nextKeyframe;
2263 if (aTrack == TrackType::kVideoTrack &&
2264 NS_SUCCEEDED(
2265 decoder.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe)) &&
2266 !nextKeyframe.IsInfinite()) {
2267 SkipVideoDemuxToNextKeyFrame(
2268 decoder.mLastDecodedSampleTime.refOr(TimeInterval()).Length());
2269 } else if (aTrack == TrackType::kAudioTrack) {
2270 decoder.Flush();
2271 } else {
2272 DDLOG(DDLogCategory::Log, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR);
2273 // We can't recover from this error.
2274 NotifyError(aTrack, NS_ERROR_DOM_MEDIA_FATAL_ERR);
2275 }
2276 return;
2277 }
2278
2279 bool needInput = NeedInput(decoder);
2280
2281 LOGV("Update(%s) ni=%d no=%d in:%" PRIu64 " out:%" PRIu64
2282 " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2283 "ds:%d sid:%u waitcdm:%d",
2284 TrackTypeToStr(aTrack), needInput, needOutput, decoder.mNumSamplesInput,
2285 decoder.mNumSamplesOutput, uint32_t(size_t(decoder.mSizeOfQueue)),
2286 decoder.mDecodeRequest.Exists(), decoder.mFlushing,
2287 decoder.mDescription.get(), uint32_t(decoder.mOutput.Length()),
2288 decoder.mWaitingForData, decoder.mDemuxEOS, int32_t(decoder.mDrainState),
2289 decoder.mLastStreamSourceID, IsDecoderWaitingForCDM(aTrack));
2290
2291 if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack)) {
2292 // If the content is encrypted, MFR won't start to create decoder until
2293 // CDMProxy is set.
2294 return;
2295 }
2296
2297 if ((decoder.IsWaitingForData() &&
2298 (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) ||
2299 (decoder.IsWaitingForKey())) {
2300 // Nothing more we can do at present.
2301 LOGV("Still waiting for data or key. data(%d)/key(%d)",
2302 decoder.mWaitingForData, decoder.mWaitingForKey);
2303 return;
2304 }
2305
2306 if (decoder.CancelWaitingForKey()) {
2307 LOGV("No longer waiting for key. Resolving waiting promise");
2308 return;
2309 }
2310
2311 if (!needInput) {
2312 LOGV("No need for additional input (pending:%u)",
2313 uint32_t(decoder.mOutput.Length()));
2314 return;
2315 }
2316
2317 // Demux samples if we don't have some.
2318 RequestDemuxSamples(aTrack);
2319
2320 HandleDemuxedSamples(aTrack, a);
2321 }
2322
ReturnOutput(MediaData * aData,TrackType aTrack)2323 void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
2324 AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK);
2325 MOZ_ASSERT(GetDecoderData(aTrack).HasPromise());
2326 MOZ_DIAGNOSTIC_ASSERT(aData->mType != MediaData::Type::NULL_DATA);
2327 LOG("Resolved data promise for %s [%" PRId64 ", %" PRId64 "]",
2328 TrackTypeToStr(aTrack), aData->mTime.ToMicroseconds(),
2329 aData->GetEndTime().ToMicroseconds());
2330
2331 if (aTrack == TrackInfo::kAudioTrack) {
2332 AudioData* audioData = aData->As<AudioData>();
2333
2334 if (audioData->mChannels != mInfo.mAudio.mChannels ||
2335 audioData->mRate != mInfo.mAudio.mRate) {
2336 LOG("change of audio format (rate:%d->%d). "
2337 "This is an unsupported configuration",
2338 mInfo.mAudio.mRate, audioData->mRate);
2339 mInfo.mAudio.mRate = audioData->mRate;
2340 mInfo.mAudio.mChannels = audioData->mChannels;
2341 MutexAutoLock lock(mAudio.mMutex);
2342 mAudio.mWorkingInfo->GetAsAudioInfo()->mRate = audioData->mRate;
2343 mAudio.mWorkingInfo->GetAsAudioInfo()->mChannels = audioData->mChannels;
2344 }
2345 mAudio.ResolvePromise(audioData, __func__);
2346 } else if (aTrack == TrackInfo::kVideoTrack) {
2347 VideoData* videoData = aData->As<VideoData>();
2348
2349 if (videoData->mDisplay != mInfo.mVideo.mDisplay) {
2350 LOG("change of video display size (%dx%d->%dx%d)",
2351 mInfo.mVideo.mDisplay.width, mInfo.mVideo.mDisplay.height,
2352 videoData->mDisplay.width, videoData->mDisplay.height);
2353 mInfo.mVideo.mDisplay = videoData->mDisplay;
2354 MutexAutoLock lock(mVideo.mMutex);
2355 mVideo.mWorkingInfo->GetAsVideoInfo()->mDisplay = videoData->mDisplay;
2356 }
2357
2358 TimeUnit nextKeyframe;
2359 if (!mVideo.HasInternalSeekPending() &&
2360 NS_SUCCEEDED(
2361 mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe))) {
2362 videoData->SetNextKeyFrameTime(nextKeyframe);
2363 }
2364
2365 mVideo.ResolvePromise(videoData, __func__);
2366 }
2367 }
2368
SizeOfVideoQueueInFrames()2369 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2370 return SizeOfQueue(TrackInfo::kVideoTrack);
2371 }
2372
SizeOfAudioQueueInFrames()2373 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2374 return SizeOfQueue(TrackInfo::kAudioTrack);
2375 }
2376
SizeOfQueue(TrackType aTrack)2377 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack) {
2378 auto& decoder = GetDecoderData(aTrack);
2379 return decoder.mSizeOfQueue;
2380 }
2381
WaitForData(MediaData::Type aType)2382 RefPtr<MediaFormatReader::WaitForDataPromise> MediaFormatReader::WaitForData(
2383 MediaData::Type aType) {
2384 MOZ_ASSERT(OnTaskQueue());
2385 TrackType trackType = aType == MediaData::Type::VIDEO_DATA
2386 ? TrackType::kVideoTrack
2387 : TrackType::kAudioTrack;
2388 auto& decoder = GetDecoderData(trackType);
2389 if (!decoder.IsWaitingForData() && !decoder.IsWaitingForKey()) {
2390 // We aren't waiting for anything.
2391 return WaitForDataPromise::CreateAndResolve(decoder.mType, __func__);
2392 }
2393 RefPtr<WaitForDataPromise> p = decoder.mWaitingPromise.Ensure(__func__);
2394 ScheduleUpdate(trackType);
2395 return p;
2396 }
2397
ResetDecode(TrackSet aTracks)2398 nsresult MediaFormatReader::ResetDecode(TrackSet aTracks) {
2399 AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK);
2400 MOZ_ASSERT(OnTaskQueue());
2401 LOGV("");
2402
2403 mSeekPromise.RejectIfExists(NS_OK, __func__);
2404 mSkipRequest.DisconnectIfExists();
2405
2406 // Do the same for any data wait promises.
2407 if (aTracks.contains(TrackInfo::kAudioTrack)) {
2408 mAudio.mWaitingPromise.RejectIfExists(
2409 WaitForDataRejectValue(MediaData::Type::AUDIO_DATA,
2410 WaitForDataRejectValue::CANCELED),
2411 __func__);
2412 }
2413
2414 if (aTracks.contains(TrackInfo::kVideoTrack)) {
2415 mVideo.mWaitingPromise.RejectIfExists(
2416 WaitForDataRejectValue(MediaData::Type::VIDEO_DATA,
2417 WaitForDataRejectValue::CANCELED),
2418 __func__);
2419 }
2420
2421 // Reset miscellaneous seeking state.
2422 mPendingSeekTime.reset();
2423
2424 if (HasVideo() && aTracks.contains(TrackInfo::kVideoTrack)) {
2425 mVideo.ResetDemuxer();
2426 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2427 Reset(TrackInfo::kVideoTrack);
2428 if (mVideo.HasPromise()) {
2429 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2430 }
2431 }
2432
2433 if (HasAudio() && aTracks.contains(TrackInfo::kAudioTrack)) {
2434 mAudio.ResetDemuxer();
2435 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2436 Reset(TrackInfo::kAudioTrack);
2437 if (mAudio.HasPromise()) {
2438 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2439 }
2440 }
2441
2442 return NS_OK;
2443 }
2444
Reset(TrackType aTrack)2445 void MediaFormatReader::Reset(TrackType aTrack) {
2446 MOZ_ASSERT(OnTaskQueue());
2447 LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack));
2448
2449 auto& decoder = GetDecoderData(aTrack);
2450
2451 decoder.ResetState();
2452 decoder.Flush();
2453
2454 LOG("Reset(%s) END", TrackTypeToStr(aTrack));
2455 }
2456
DropDecodedSamples(TrackType aTrack)2457 void MediaFormatReader::DropDecodedSamples(TrackType aTrack) {
2458 MOZ_ASSERT(OnTaskQueue());
2459 auto& decoder = GetDecoderData(aTrack);
2460 size_t lengthDecodedQueue = decoder.mOutput.Length();
2461 if (lengthDecodedQueue && decoder.mTimeThreshold.isSome()) {
2462 auto time = decoder.mOutput.LastElement()->mTime;
2463 if (time >= decoder.mTimeThreshold.ref().Time()) {
2464 // We would have reached our internal seek target.
2465 decoder.mTimeThreshold.reset();
2466 }
2467 }
2468 decoder.mOutput.Clear();
2469 decoder.mSizeOfQueue -= lengthDecodedQueue;
2470 if (aTrack == TrackInfo::kVideoTrack && mFrameStats) {
2471 mFrameStats->Accumulate({0, 0, 0, lengthDecodedQueue, 0, 0});
2472 }
2473 }
2474
SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold)2475 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold) {
2476 AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2477 MEDIA_PLAYBACK);
2478 MOZ_ASSERT(OnTaskQueue());
2479 LOG("Skipping up to %" PRId64, aTimeThreshold.ToMicroseconds());
2480
2481 // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2482 // As such we can drop all already decoded samples and discard all pending
2483 // samples.
2484 DropDecodedSamples(TrackInfo::kVideoTrack);
2485
2486 mVideo.mTrackDemuxer->SkipToNextRandomAccessPoint(aTimeThreshold)
2487 ->Then(OwnerThread(), __func__, this,
2488 &MediaFormatReader::OnVideoSkipCompleted,
2489 &MediaFormatReader::OnVideoSkipFailed)
2490 ->Track(mSkipRequest);
2491 }
2492
VideoSkipReset(uint32_t aSkipped)2493 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped) {
2494 PROFILER_ADD_MARKER("SkippedVideoDecode", MEDIA_PLAYBACK);
2495 MOZ_ASSERT(OnTaskQueue());
2496
2497 // Some frames may have been output by the decoder since we initiated the
2498 // videoskip process and we know they would be late.
2499 DropDecodedSamples(TrackInfo::kVideoTrack);
2500 // Report the pending frames as dropped.
2501 if (mFrameStats) {
2502 uint32_t droppedDecoderCount = SizeOfVideoQueueInFrames();
2503 mFrameStats->Accumulate({0, 0, 0, droppedDecoderCount, 0, 0});
2504 }
2505
2506 // Cancel any pending demux request and pending demuxed samples.
2507 mVideo.mDemuxRequest.DisconnectIfExists();
2508 Reset(TrackType::kVideoTrack);
2509
2510 if (mFrameStats) {
2511 mFrameStats->Accumulate({aSkipped, 0, 0, aSkipped, 0, 0});
2512 }
2513
2514 mVideo.mNumSamplesSkippedTotal += aSkipped;
2515 }
2516
OnVideoSkipCompleted(uint32_t aSkipped)2517 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped) {
2518 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2519 MEDIA_PLAYBACK);
2520 MOZ_ASSERT(OnTaskQueue());
2521 LOG("Skipping succeeded, skipped %u frames", aSkipped);
2522 mSkipRequest.Complete();
2523
2524 DDLOG(DDLogCategory::Log, "video_skipped", DDNoValue());
2525
2526 VideoSkipReset(aSkipped);
2527
2528 ScheduleUpdate(TrackInfo::kVideoTrack);
2529 }
2530
OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure)2531 void MediaFormatReader::OnVideoSkipFailed(
2532 MediaTrackDemuxer::SkipFailureHolder aFailure) {
2533 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK);
2534 MOZ_ASSERT(OnTaskQueue());
2535 LOG("Skipping failed, skipped %u frames", aFailure.mSkipped);
2536 mSkipRequest.Complete();
2537
2538 switch (aFailure.mFailure.Code()) {
2539 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2540 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2541 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2542 aFailure.mFailure);
2543 // Some frames may have been output by the decoder since we initiated the
2544 // videoskip process and we know they would be late.
2545 DropDecodedSamples(TrackInfo::kVideoTrack);
2546 // We can't complete the skip operation, will just service a video frame
2547 // normally.
2548 ScheduleUpdate(TrackInfo::kVideoTrack);
2549 break;
2550 case NS_ERROR_DOM_MEDIA_CANCELED:
2551 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2552 aFailure.mFailure);
2553 if (mVideo.HasPromise()) {
2554 mVideo.RejectPromise(aFailure.mFailure, __func__);
2555 }
2556 break;
2557 default:
2558 DDLOG(DDLogCategory::Log, "video_skipping_error", aFailure.mFailure);
2559 NotifyError(TrackType::kVideoTrack, aFailure.mFailure);
2560 break;
2561 }
2562 }
2563
Seek(const SeekTarget & aTarget)2564 RefPtr<MediaFormatReader::SeekPromise> MediaFormatReader::Seek(
2565 const SeekTarget& aTarget) {
2566 AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK);
2567 MOZ_ASSERT(OnTaskQueue());
2568
2569 LOG("aTarget=(%" PRId64 ")", aTarget.GetTime().ToMicroseconds());
2570
2571 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty());
2572 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2573 MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() || !mAudio.HasPromise());
2574 MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime.isNothing());
2575 MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2576 MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() ||
2577 mAudio.mTimeThreshold.isNothing());
2578
2579 if (!mInfo.mMediaSeekable && !mInfo.mMediaSeekableOnlyInBufferedRanges) {
2580 LOG("Seek() END (Unseekable)");
2581 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2582 }
2583
2584 if (mShutdown) {
2585 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2586 }
2587
2588 SetSeekTarget(aTarget);
2589
2590 RefPtr<SeekPromise> p = mSeekPromise.Ensure(__func__);
2591
2592 ScheduleSeek();
2593
2594 return p;
2595 }
2596
SetSeekTarget(const SeekTarget & aTarget)2597 void MediaFormatReader::SetSeekTarget(const SeekTarget& aTarget) {
2598 MOZ_ASSERT(OnTaskQueue());
2599
2600 mOriginalSeekTarget = aTarget;
2601 mFallbackSeekTime = mPendingSeekTime = Some(aTarget.GetTime());
2602 }
2603
ScheduleSeek()2604 void MediaFormatReader::ScheduleSeek() {
2605 if (mSeekScheduled) {
2606 return;
2607 }
2608 mSeekScheduled = true;
2609 nsresult rv = OwnerThread()->Dispatch(NewRunnableMethod(
2610 "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek));
2611 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
2612 Unused << rv;
2613 }
2614
AttemptSeek()2615 void MediaFormatReader::AttemptSeek() {
2616 AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK);
2617 MOZ_ASSERT(OnTaskQueue());
2618
2619 mSeekScheduled = false;
2620
2621 if (mPendingSeekTime.isNothing()) {
2622 return;
2623 }
2624
2625 if (HasVideo()) {
2626 mVideo.ResetDemuxer();
2627 mVideo.ResetState();
2628 }
2629
2630 // Don't reset the audio demuxer not state when seeking video only
2631 // as it will cause the audio to seek back to the beginning
2632 // resulting in out-of-sync audio from video.
2633 if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
2634 mAudio.ResetDemuxer();
2635 mAudio.ResetState();
2636 }
2637
2638 if (HasVideo()) {
2639 DoVideoSeek();
2640 } else if (HasAudio()) {
2641 DoAudioSeek();
2642 } else {
2643 MOZ_CRASH();
2644 }
2645 }
2646
OnSeekFailed(TrackType aTrack,const MediaResult & aError)2647 void MediaFormatReader::OnSeekFailed(TrackType aTrack,
2648 const MediaResult& aError) {
2649 AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK);
2650 MOZ_ASSERT(OnTaskQueue());
2651 LOGV("%s failure:%s", TrackTypeToStr(aTrack), aError.ErrorName().get());
2652 if (aTrack == TrackType::kVideoTrack) {
2653 mVideo.mSeekRequest.Complete();
2654 } else {
2655 mAudio.mSeekRequest.Complete();
2656 }
2657
2658 if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
2659 if (HasVideo() && aTrack == TrackType::kAudioTrack &&
2660 mFallbackSeekTime.isSome() &&
2661 mPendingSeekTime.ref() != mFallbackSeekTime.ref()) {
2662 // We have failed to seek audio where video seeked to earlier.
2663 // Attempt to seek instead to the closest point that we know we have in
2664 // order to limit A/V sync discrepency.
2665
2666 // Ensure we have the most up to date buffered ranges.
2667 UpdateReceivedNewData(TrackType::kAudioTrack);
2668 Maybe<TimeUnit> nextSeekTime;
2669 // Find closest buffered time found after video seeked time.
2670 for (const auto& timeRange : mAudio.mTimeRanges) {
2671 if (timeRange.mStart >= mPendingSeekTime.ref()) {
2672 nextSeekTime.emplace(timeRange.mStart);
2673 break;
2674 }
2675 }
2676 if (nextSeekTime.isNothing() ||
2677 nextSeekTime.ref() > mFallbackSeekTime.ref()) {
2678 nextSeekTime = Some(mFallbackSeekTime.ref());
2679 LOG("Unable to seek audio to video seek time. A/V sync may be broken");
2680 } else {
2681 mFallbackSeekTime.reset();
2682 }
2683 mPendingSeekTime = nextSeekTime;
2684 DoAudioSeek();
2685 return;
2686 }
2687 NotifyWaitingForData(aTrack);
2688 }
2689 MOZ_ASSERT(!mVideo.mSeekRequest.Exists() && !mAudio.mSeekRequest.Exists());
2690 mPendingSeekTime.reset();
2691
2692 auto type = aTrack == TrackType::kAudioTrack ? MediaData::Type::AUDIO_DATA
2693 : MediaData::Type::VIDEO_DATA;
2694 mSeekPromise.Reject(SeekRejectValue(type, aError), __func__);
2695 }
2696
DoVideoSeek()2697 void MediaFormatReader::DoVideoSeek() {
2698 AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK);
2699 MOZ_ASSERT(mPendingSeekTime.isSome());
2700 LOGV("Seeking video to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
2701 auto seekTime = mPendingSeekTime.ref();
2702 mVideo.mTrackDemuxer->Seek(seekTime)
2703 ->Then(OwnerThread(), __func__, this,
2704 &MediaFormatReader::OnVideoSeekCompleted,
2705 &MediaFormatReader::OnVideoSeekFailed)
2706 ->Track(mVideo.mSeekRequest);
2707 }
2708
OnVideoSeekCompleted(TimeUnit aTime)2709 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime) {
2710 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
2711 MEDIA_PLAYBACK);
2712 MOZ_ASSERT(OnTaskQueue());
2713 LOGV("Video seeked to %" PRId64, aTime.ToMicroseconds());
2714 mVideo.mSeekRequest.Complete();
2715
2716 mVideo.mFirstFrameTime = Some(aTime);
2717 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2718
2719 SetVideoDecodeThreshold();
2720
2721 if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
2722 MOZ_ASSERT(mPendingSeekTime.isSome());
2723 if (mOriginalSeekTarget.IsFast()) {
2724 // We are performing a fast seek. We need to seek audio to where the
2725 // video seeked to, to ensure proper A/V sync once playback resume.
2726 mPendingSeekTime = Some(aTime);
2727 }
2728 DoAudioSeek();
2729 } else {
2730 mPendingSeekTime.reset();
2731 mSeekPromise.Resolve(aTime, __func__);
2732 }
2733 }
2734
OnVideoSeekFailed(const MediaResult & aError)2735 void MediaFormatReader::OnVideoSeekFailed(const MediaResult& aError) {
2736 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK);
2737 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2738 OnSeekFailed(TrackType::kVideoTrack, aError);
2739 }
2740
SetVideoDecodeThreshold()2741 void MediaFormatReader::SetVideoDecodeThreshold() {
2742 MOZ_ASSERT(OnTaskQueue());
2743
2744 if (!HasVideo() || !mVideo.mDecoder) {
2745 return;
2746 }
2747
2748 if (!mVideo.mTimeThreshold && !IsSeeking()) {
2749 return;
2750 }
2751
2752 TimeUnit threshold;
2753 if (mVideo.mTimeThreshold) {
2754 // For internalSeek.
2755 threshold = mVideo.mTimeThreshold.ref().Time();
2756 } else if (IsSeeking()) {
2757 // If IsSeeking() is true, then video seek must have completed already.
2758 TimeUnit keyframe;
2759 if (NS_FAILED(mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&keyframe))) {
2760 return;
2761 }
2762
2763 // If the key frame is invalid/infinite, it means the target position is
2764 // closing to end of stream. We don't want to skip any frame at this point.
2765 threshold = keyframe.IsValid() && !keyframe.IsInfinite()
2766 ? mOriginalSeekTarget.GetTime()
2767 : TimeUnit::Invalid();
2768 } else {
2769 return;
2770 }
2771
2772 if (threshold.IsValid()) {
2773 LOG("Set seek threshold to %" PRId64, threshold.ToMicroseconds());
2774 } else {
2775 LOG("Resetting seek threshold");
2776 }
2777 mVideo.mDecoder->SetSeekThreshold(threshold);
2778 }
2779
DoAudioSeek()2780 void MediaFormatReader::DoAudioSeek() {
2781 AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK);
2782 MOZ_ASSERT(mPendingSeekTime.isSome());
2783 LOGV("Seeking audio to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
2784 auto seekTime = mPendingSeekTime.ref();
2785 mAudio.mTrackDemuxer->Seek(seekTime)
2786 ->Then(OwnerThread(), __func__, this,
2787 &MediaFormatReader::OnAudioSeekCompleted,
2788 &MediaFormatReader::OnAudioSeekFailed)
2789 ->Track(mAudio.mSeekRequest);
2790 }
2791
OnAudioSeekCompleted(TimeUnit aTime)2792 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime) {
2793 MOZ_ASSERT(OnTaskQueue());
2794 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
2795 MEDIA_PLAYBACK);
2796 LOGV("Audio seeked to %" PRId64, aTime.ToMicroseconds());
2797 mAudio.mSeekRequest.Complete();
2798 mAudio.mFirstFrameTime = Some(aTime);
2799 mPendingSeekTime.reset();
2800 mSeekPromise.Resolve(aTime, __func__);
2801 }
2802
OnAudioSeekFailed(const MediaResult & aError)2803 void MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError) {
2804 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK);
2805 OnSeekFailed(TrackType::kAudioTrack, aError);
2806 }
2807
ReleaseResources()2808 void MediaFormatReader::ReleaseResources() {
2809 LOGV("");
2810 if (mShutdown) {
2811 return;
2812 }
2813 ShutdownDecoder(TrackInfo::kAudioTrack);
2814 ShutdownDecoder(TrackInfo::kVideoTrack);
2815 }
2816
VideoIsHardwareAccelerated() const2817 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
2818 return mVideo.mIsHardwareAccelerated;
2819 }
2820
NotifyTrackDemuxers()2821 void MediaFormatReader::NotifyTrackDemuxers() {
2822 MOZ_ASSERT(OnTaskQueue());
2823
2824 LOGV("");
2825
2826 if (!mInitDone) {
2827 return;
2828 }
2829
2830 if (HasVideo()) {
2831 mVideo.mReceivedNewData = true;
2832 ScheduleUpdate(TrackType::kVideoTrack);
2833 }
2834 if (HasAudio()) {
2835 mAudio.mReceivedNewData = true;
2836 ScheduleUpdate(TrackType::kAudioTrack);
2837 }
2838 }
2839
NotifyDataArrived()2840 void MediaFormatReader::NotifyDataArrived() {
2841 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK);
2842 MOZ_ASSERT(OnTaskQueue());
2843
2844 if (mShutdown || !mDemuxer || !mDemuxerInitDone) {
2845 return;
2846 }
2847
2848 if (mNotifyDataArrivedPromise.Exists()) {
2849 // Already one in progress. Set the dirty flag so we can process it later.
2850 mPendingNotifyDataArrived = true;
2851 return;
2852 }
2853
2854 RefPtr<MediaFormatReader> self = this;
2855 mDemuxer->NotifyDataArrived()
2856 ->Then(
2857 OwnerThread(), __func__,
2858 [self]() {
2859 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
2860 MEDIA_PLAYBACK);
2861 self->mNotifyDataArrivedPromise.Complete();
2862 self->UpdateBuffered();
2863 self->NotifyTrackDemuxers();
2864 if (self->mPendingNotifyDataArrived) {
2865 self->mPendingNotifyDataArrived = false;
2866 self->NotifyDataArrived();
2867 }
2868 },
2869 [self]() { self->mNotifyDataArrivedPromise.Complete(); })
2870 ->Track(mNotifyDataArrivedPromise);
2871 }
2872
UpdateBuffered()2873 void MediaFormatReader::UpdateBuffered() {
2874 AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK);
2875 MOZ_ASSERT(OnTaskQueue());
2876
2877 if (mShutdown) {
2878 return;
2879 }
2880
2881 if (!mInitDone || !mHasStartTime) {
2882 mBuffered = TimeIntervals();
2883 return;
2884 }
2885
2886 if (HasVideo()) {
2887 mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered();
2888 bool hasLastEnd;
2889 auto lastEnd = mVideo.mTimeRanges.GetEnd(&hasLastEnd);
2890 if (hasLastEnd) {
2891 if (mVideo.mLastTimeRangesEnd &&
2892 mVideo.mLastTimeRangesEnd.ref() < lastEnd) {
2893 // New data was added after our previous end, we can clear the EOS flag.
2894 mVideo.mDemuxEOS = false;
2895 ScheduleUpdate(TrackInfo::kVideoTrack);
2896 }
2897 mVideo.mLastTimeRangesEnd = Some(lastEnd);
2898 }
2899 }
2900 if (HasAudio()) {
2901 mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered();
2902 bool hasLastEnd;
2903 auto lastEnd = mAudio.mTimeRanges.GetEnd(&hasLastEnd);
2904 if (hasLastEnd) {
2905 if (mAudio.mLastTimeRangesEnd &&
2906 mAudio.mLastTimeRangesEnd.ref() < lastEnd) {
2907 // New data was added after our previous end, we can clear the EOS flag.
2908 mAudio.mDemuxEOS = false;
2909 ScheduleUpdate(TrackInfo::kAudioTrack);
2910 }
2911 mAudio.mLastTimeRangesEnd = Some(lastEnd);
2912 }
2913 }
2914
2915 media::TimeIntervals intervals;
2916 if (HasAudio() && HasVideo()) {
2917 intervals = media::Intersection(mVideo.mTimeRanges, mAudio.mTimeRanges);
2918 } else if (HasAudio()) {
2919 intervals = mAudio.mTimeRanges;
2920 } else if (HasVideo()) {
2921 intervals = mVideo.mTimeRanges;
2922 }
2923
2924 if (intervals.IsEmpty() || intervals.GetStart() == TimeUnit::Zero()) {
2925 // IntervalSet already starts at 0 or is empty, nothing to shift.
2926 mBuffered = intervals;
2927 } else {
2928 mBuffered = intervals.Shift(TimeUnit::Zero() - mInfo.mStartTime);
2929 }
2930 }
2931
GetImageContainer()2932 layers::ImageContainer* MediaFormatReader::GetImageContainer() {
2933 return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
2934 : nullptr;
2935 }
2936
GetDebugInfo(dom::MediaFormatReaderDebugInfo & aInfo)2937 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo) {
2938 nsCString result;
2939 nsAutoCString audioDecoderName("unavailable");
2940 nsAutoCString videoDecoderName = audioDecoderName;
2941 nsAutoCString audioType("none");
2942 nsAutoCString videoType("none");
2943
2944 AudioInfo audioInfo;
2945 {
2946 MutexAutoLock lock(mAudio.mMutex);
2947 if (HasAudio()) {
2948 audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
2949 audioDecoderName = mAudio.mDecoder ? mAudio.mDecoder->GetDescriptionName()
2950 : mAudio.mDescription;
2951 audioType = audioInfo.mMimeType;
2952 }
2953 }
2954
2955 VideoInfo videoInfo;
2956 {
2957 MutexAutoLock lock(mVideo.mMutex);
2958 if (HasVideo()) {
2959 videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
2960 videoDecoderName = mVideo.mDecoder ? mVideo.mDecoder->GetDescriptionName()
2961 : mVideo.mDescription;
2962 videoType = videoInfo.mMimeType;
2963 }
2964 }
2965
2966 aInfo.mAudioDecoderName = NS_ConvertUTF8toUTF16(audioDecoderName);
2967 aInfo.mAudioType = NS_ConvertUTF8toUTF16(audioType);
2968 aInfo.mAudioChannels = audioInfo.mChannels;
2969 aInfo.mAudioRate = audioInfo.mRate / 1000.0f;
2970 aInfo.mAudioFramesDecoded = mAudio.mNumSamplesOutputTotal;
2971
2972 if (HasAudio()) {
2973 aInfo.mAudioState.mNeedInput = NeedInput(mAudio);
2974 aInfo.mAudioState.mHasPromise = mAudio.HasPromise();
2975 aInfo.mAudioState.mWaitingPromise = !mAudio.mWaitingPromise.IsEmpty();
2976 aInfo.mAudioState.mHasDemuxRequest = mAudio.mDemuxRequest.Exists();
2977 aInfo.mAudioState.mDemuxQueueSize =
2978 uint32_t(mAudio.mQueuedSamples.Length());
2979 aInfo.mAudioState.mHasDecoder = mAudio.mDecodeRequest.Exists();
2980 aInfo.mAudioState.mTimeTreshold =
2981 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().Time().ToSeconds()
2982 : -1.0;
2983 aInfo.mAudioState.mTimeTresholdHasSeeked =
2984 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().mHasSeeked : false;
2985 aInfo.mAudioState.mNumSamplesInput = mAudio.mNumSamplesInput;
2986 aInfo.mAudioState.mNumSamplesOutput = mAudio.mNumSamplesOutput;
2987 aInfo.mAudioState.mQueueSize = size_t(mAudio.mSizeOfQueue);
2988 aInfo.mAudioState.mPending = mAudio.mOutput.Length();
2989 aInfo.mAudioState.mWaitingForData = mAudio.mWaitingForData;
2990 aInfo.mAudioState.mDemuxEOS = mAudio.mDemuxEOS;
2991 aInfo.mAudioState.mDrainState = int32_t(mAudio.mDrainState);
2992 aInfo.mAudioState.mWaitingForKey = mAudio.mWaitingForKey;
2993 aInfo.mAudioState.mLastStreamSourceID = mAudio.mLastStreamSourceID;
2994 }
2995
2996 aInfo.mVideoDecoderName = NS_ConvertUTF8toUTF16(videoDecoderName);
2997 aInfo.mVideoType = NS_ConvertUTF8toUTF16(videoType);
2998 aInfo.mVideoWidth =
2999 videoInfo.mDisplay.width < 0 ? 0 : videoInfo.mDisplay.width;
3000 aInfo.mVideoHeight =
3001 videoInfo.mDisplay.height < 0 ? 0 : videoInfo.mDisplay.height;
3002 aInfo.mVideoRate = mVideo.mMeanRate.Mean();
3003 aInfo.mVideoHardwareAccelerated = VideoIsHardwareAccelerated();
3004 aInfo.mVideoNumSamplesOutputTotal = mVideo.mNumSamplesOutputTotal;
3005 aInfo.mVideoNumSamplesSkippedTotal = mVideo.mNumSamplesSkippedTotal;
3006
3007 if (HasVideo()) {
3008 aInfo.mVideoState.mNeedInput = NeedInput(mVideo);
3009 aInfo.mVideoState.mHasPromise = mVideo.HasPromise();
3010 aInfo.mVideoState.mWaitingPromise = !mVideo.mWaitingPromise.IsEmpty();
3011 aInfo.mVideoState.mHasDemuxRequest = mVideo.mDemuxRequest.Exists();
3012 aInfo.mVideoState.mDemuxQueueSize =
3013 uint32_t(mVideo.mQueuedSamples.Length());
3014 aInfo.mVideoState.mHasDecoder = mVideo.mDecodeRequest.Exists();
3015 aInfo.mVideoState.mTimeTreshold =
3016 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().Time().ToSeconds()
3017 : -1.0;
3018 aInfo.mVideoState.mTimeTresholdHasSeeked =
3019 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().mHasSeeked : false;
3020 aInfo.mVideoState.mNumSamplesInput = mVideo.mNumSamplesInput;
3021 aInfo.mVideoState.mNumSamplesOutput = mVideo.mNumSamplesOutput;
3022 aInfo.mVideoState.mQueueSize = size_t(mVideo.mSizeOfQueue);
3023 aInfo.mVideoState.mPending = mVideo.mOutput.Length();
3024 aInfo.mVideoState.mWaitingForData = mVideo.mWaitingForData;
3025 aInfo.mVideoState.mDemuxEOS = mVideo.mDemuxEOS;
3026 aInfo.mVideoState.mDrainState = int32_t(mVideo.mDrainState);
3027 aInfo.mVideoState.mWaitingForKey = mVideo.mWaitingForKey;
3028 aInfo.mVideoState.mLastStreamSourceID = mVideo.mLastStreamSourceID;
3029 }
3030
3031 // Looking at dropped frames
3032 FrameStatisticsData stats = mFrameStats->GetFrameStatisticsData();
3033 aInfo.mFrameStats.mDroppedDecodedFrames = stats.mDroppedDecodedFrames;
3034 aInfo.mFrameStats.mDroppedSinkFrames = stats.mDroppedSinkFrames;
3035 aInfo.mFrameStats.mDroppedCompositorFrames = stats.mDroppedCompositorFrames;
3036 }
3037
SetVideoNullDecode(bool aIsNullDecode)3038 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode) {
3039 MOZ_ASSERT(OnTaskQueue());
3040 return SetNullDecode(TrackType::kVideoTrack, aIsNullDecode);
3041 }
3042
UpdateCompositor(already_AddRefed<layers::KnowsCompositor> aCompositor)3043 void MediaFormatReader::UpdateCompositor(
3044 already_AddRefed<layers::KnowsCompositor> aCompositor) {
3045 MOZ_ASSERT(OnTaskQueue());
3046 mKnowsCompositor = aCompositor;
3047 }
3048
SetNullDecode(TrackType aTrack,bool aIsNullDecode)3049 void MediaFormatReader::SetNullDecode(TrackType aTrack, bool aIsNullDecode) {
3050 MOZ_ASSERT(OnTaskQueue());
3051
3052 auto& decoder = GetDecoderData(aTrack);
3053 if (decoder.mIsNullDecode == aIsNullDecode) {
3054 return;
3055 }
3056
3057 LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3058 TrackTypeToStr(aTrack), decoder.mIsNullDecode, aIsNullDecode);
3059
3060 decoder.mIsNullDecode = aIsNullDecode;
3061 ShutdownDecoder(aTrack);
3062 }
3063
OnFirstDemuxCompleted(TrackInfo::TrackType aType,RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)3064 void MediaFormatReader::OnFirstDemuxCompleted(
3065 TrackInfo::TrackType aType,
3066 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
3067 AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3068 MEDIA_PLAYBACK);
3069 MOZ_ASSERT(OnTaskQueue());
3070
3071 if (mShutdown) {
3072 return;
3073 }
3074
3075 auto& decoder = GetDecoderData(aType);
3076 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3077 decoder.mFirstDemuxedSampleTime.emplace(aSamples->GetSamples()[0]->mTime);
3078 MaybeResolveMetadataPromise();
3079 }
3080
OnFirstDemuxFailed(TrackInfo::TrackType aType,const MediaResult & aError)3081 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType,
3082 const MediaResult& aError) {
3083 MOZ_ASSERT(OnTaskQueue());
3084
3085 if (mShutdown) {
3086 return;
3087 }
3088
3089 auto& decoder = GetDecoderData(aType);
3090 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3091 decoder.mFirstDemuxedSampleTime.emplace(TimeUnit::FromInfinity());
3092 MaybeResolveMetadataPromise();
3093 }
3094
3095 } // namespace mozilla
3096
3097 #undef NS_DispatchToMainThread
3098 #undef LOGV
3099 #undef LOG
3100