1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaFormatReader.h"
8
9 #include <algorithm>
10 #include <map>
11 #include <queue>
12
13 #include "AllocationPolicy.h"
14 #ifdef MOZ_AV1
15 # include "AOMDecoder.h"
16 #endif
17 #include "DecoderBenchmark.h"
18 #include "MediaData.h"
19 #include "MediaDataDecoderProxy.h"
20 #include "MediaInfo.h"
21 #include "MP4Decoder.h"
22 #include "PDMFactory.h"
23 #include "PerformanceRecorder.h"
24 #include "VideoFrameContainer.h"
25 #include "VideoUtils.h"
26 #include "VPXDecoder.h"
27 #include "mozilla/AbstractThread.h"
28 #include "mozilla/CDMProxy.h"
29 #include "mozilla/ClearOnShutdown.h"
30 #include "mozilla/NotNull.h"
31 #include "mozilla/Preferences.h"
32 #include "mozilla/ProfilerLabels.h"
33 #include "mozilla/ProfilerMarkers.h"
34 #include "mozilla/SharedThreadPool.h"
35 #include "mozilla/StaticPrefs_media.h"
36 #include "mozilla/TaskQueue.h"
37 #include "mozilla/Unused.h"
38 #include "nsContentUtils.h"
39 #include "nsPrintfCString.h"
40 #include "nsTHashSet.h"
41
42 using namespace mozilla::media;
43
44 static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
45 mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
46
47 #define LOG(arg, ...) \
48 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
49 __func__, ##__VA_ARGS__)
50 #define LOGV(arg, ...) \
51 DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
52 __func__, ##__VA_ARGS__)
53
54 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
55
56 namespace mozilla {
57
58 typedef void* MediaDataDecoderID;
59
60 /**
61 * This class tracks shutdown promises to ensure all decoders are shut down
62 * completely before MFR continues the rest of the shutdown procedure.
63 */
64 class MediaFormatReader::ShutdownPromisePool {
65 public:
ShutdownPromisePool()66 ShutdownPromisePool()
67 : mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}
68
69 // Return a promise which will be resolved when all the tracking promises
70 // are resolved. Note no more promises should be added for tracking once
71 // this function is called.
72 RefPtr<ShutdownPromise> Shutdown();
73
74 // Track a shutdown promise.
75 void Track(RefPtr<ShutdownPromise> aPromise);
76
77 // Shut down a decoder and track its shutdown promise.
ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder)78 void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {
79 Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());
80 }
81
82 private:
83 bool mShutdown = false;
84 const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;
85 nsTHashSet<RefPtr<ShutdownPromise>> mPromises;
86 };
87
Shutdown()88 RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {
89 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
90 mShutdown = true;
91 if (mPromises.Count() == 0) {
92 mOnShutdownComplete->Resolve(true, __func__);
93 }
94 return mOnShutdownComplete;
95 }
96
Track(RefPtr<ShutdownPromise> aPromise)97 void MediaFormatReader::ShutdownPromisePool::Track(
98 RefPtr<ShutdownPromise> aPromise) {
99 MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
100 MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));
101 mPromises.Insert(aPromise);
102 aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {
103 MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));
104 mPromises.Remove(aPromise);
105 if (mShutdown && mPromises.Count() == 0) {
106 mOnShutdownComplete->Resolve(true, __func__);
107 }
108 });
109 }
110
ShutdownDecoder()111 void MediaFormatReader::DecoderData::ShutdownDecoder() {
112 MOZ_ASSERT(mOwner->OnTaskQueue());
113
114 MutexAutoLock lock(mMutex);
115
116 if (!mDecoder) {
117 // No decoder to shut down.
118 return;
119 }
120
121 if (mFlushing) {
122 // Flush is is in action. Shutdown will be initiated after flush completes.
123 MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);
124 mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));
125 // The order of decoder creation and shutdown is handled by LocalAllocPolicy
126 // and ShutdownPromisePool. MFR can now reset these members to a fresh state
127 // and be ready to create new decoders again without explicitly waiting for
128 // flush/shutdown to complete.
129 mShutdownPromise = nullptr;
130 mFlushing = false;
131 } else {
132 // No flush is in action. We can shut down the decoder now.
133 mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());
134 }
135
136 // mShutdownPromisePool will handle the order of decoder shutdown so
137 // we can forget mDecoder and be ready to create a new one.
138 mDecoder = nullptr;
139 mDescription = "shutdown"_ns;
140 mHasReportedVideoHardwareSupportTelemtry = false;
141 mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA
142 ? TrackType::kAudioTrack
143 : TrackType::kVideoTrack);
144 }
145
Flush()146 void MediaFormatReader::DecoderData::Flush() {
147 AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);
148 MOZ_ASSERT(mOwner->OnTaskQueue());
149
150 if (mFlushing || mFlushed) {
151 // Flush still pending or already flushed, nothing more to do.
152 return;
153 }
154 mDecodeRequest.DisconnectIfExists();
155 mDrainRequest.DisconnectIfExists();
156 mDrainState = DrainState::None;
157 CancelWaitingForKey();
158 mOutput.Clear();
159 mNumSamplesInput = 0;
160 mNumSamplesOutput = 0;
161 mSizeOfQueue = 0;
162 if (mDecoder) {
163 TrackType type = mType == MediaData::Type::AUDIO_DATA
164 ? TrackType::kAudioTrack
165 : TrackType::kVideoTrack;
166 mFlushing = true;
167 MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);
168 mShutdownPromise = new SharedShutdownPromiseHolder();
169 RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;
170 RefPtr<MediaDataDecoder> d = mDecoder;
171 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
172 "flushing", DDNoValue{});
173 mDecoder->Flush()->Then(
174 mOwner->OwnerThread(), __func__,
175 [type, this, p, d]() {
176 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
177 MEDIA_PLAYBACK);
178 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
179 "flushed", DDNoValue{});
180 if (!p->IsEmpty()) {
181 // Shutdown happened before flush completes.
182 // Let's continue to shut down the decoder. Note
183 // we don't access |this| because this decoder
184 // is no longer managed by MFR::DecoderData.
185 d->Shutdown()->ChainTo(p->Steal(), __func__);
186 return;
187 }
188 mFlushing = false;
189 mShutdownPromise = nullptr;
190 mOwner->ScheduleUpdate(type);
191 },
192 [type, this, p, d](const MediaResult& aError) {
193 AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
194 MEDIA_PLAYBACK);
195 DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
196 "flush_error", aError);
197 if (!p->IsEmpty()) {
198 d->Shutdown()->ChainTo(p->Steal(), __func__);
199 return;
200 }
201 mFlushing = false;
202 mShutdownPromise = nullptr;
203 mOwner->NotifyError(type, aError);
204 });
205 }
206 mFlushed = true;
207 }
208
209 class MediaFormatReader::DecoderFactory {
210 using InitPromise = MediaDataDecoder::InitPromise;
211 using TokenPromise = AllocPolicy::Promise;
212 using Token = AllocPolicy::Token;
213 using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise;
214
215 public:
DecoderFactory(MediaFormatReader * aOwner)216 explicit DecoderFactory(MediaFormatReader* aOwner)
217 : mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),
218 mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),
219 mOwner(WrapNotNull(aOwner)) {
220 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
221 this);
222 DecoderDoctorLogger::LinkParentAndChild(
223 aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);
224 }
225
~DecoderFactory()226 ~DecoderFactory() {
227 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
228 this);
229 }
230
231 void CreateDecoder(TrackType aTrack);
232
233 // Shutdown any decoder pending initialization and reset mAudio/mVideo to its
234 // pristine state so CreateDecoder() is ready to be called again immediately.
ShutdownDecoder(TrackType aTrack)235 void ShutdownDecoder(TrackType aTrack) {
236 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
237 aTrack == TrackInfo::kVideoTrack);
238 auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
239 data.mPolicy->Cancel();
240 data.mTokenRequest.DisconnectIfExists();
241 if (data.mLiveToken) {
242 // We haven't completed creation of the decoder, and it hasn't been
243 // initialised yet.
244 data.mLiveToken = nullptr;
245 // The decoder will be shutdown as soon as it's available and tracked by
246 // the ShutdownPromisePool.
247 mOwner->mShutdownPromisePool->Track(data.mCreateDecoderPromise->Then(
248 mOwner->mTaskQueue, __func__,
249 [](CreateDecoderPromise::ResolveOrRejectValue&& aResult) {
250 if (aResult.IsReject()) {
251 return ShutdownPromise::CreateAndResolve(true, __func__);
252 }
253 return aResult.ResolveValue()->Shutdown();
254 }));
255 // Free the token to leave room for a new decoder.
256 data.mToken = nullptr;
257 }
258 data.mInitRequest.DisconnectIfExists();
259 if (data.mDecoder) {
260 mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());
261 }
262 data.mStage = Stage::None;
263 MOZ_ASSERT(!data.mToken);
264 }
265
266 private:
267 enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };
268
269 struct Data {
Datamozilla::MediaFormatReader::DecoderFactory::Data270 Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)
271 : mOwnerData(aOwnerData),
272 mTrack(aTrack),
273 mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}
274 DecoderData& mOwnerData;
275 const TrackType mTrack;
276 RefPtr<SingleAllocPolicy> mPolicy;
277 Stage mStage = Stage::None;
278 RefPtr<Token> mToken;
279 RefPtr<MediaDataDecoder> mDecoder;
280 MozPromiseRequestHolder<TokenPromise> mTokenRequest;
281 struct DecoderCancelled : public SupportsWeakPtr {
282 NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled)
283 private:
284 ~DecoderCancelled() = default;
285 };
286 // Set when decoder is about to be created. If cleared before the decoder
287 // creation promise is resolved; it indicates that Shutdown() was called and
288 // further processing such as initialization should stop.
289 RefPtr<DecoderCancelled> mLiveToken;
290 RefPtr<CreateDecoderPromise> mCreateDecoderPromise;
291 MozPromiseRequestHolder<InitPromise> mInitRequest;
292 } mAudio, mVideo;
293
294 void RunStage(Data& aData);
295 void DoCreateDecoder(Data& aData);
296 void DoInitDecoder(Data& aData);
297
298 // guaranteed to be valid by the owner.
299 const NotNull<MediaFormatReader*> mOwner;
300 };
301
CreateDecoder(TrackType aTrack)302 void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {
303 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
304 aTrack == TrackInfo::kVideoTrack);
305 RunStage(aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo);
306 }
307
RunStage(Data & aData)308 void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {
309 switch (aData.mStage) {
310 case Stage::None: {
311 MOZ_ASSERT(!aData.mToken);
312 aData.mPolicy->Alloc()
313 ->Then(
314 mOwner->OwnerThread(), __func__,
315 [this, &aData](RefPtr<Token> aToken) {
316 aData.mTokenRequest.Complete();
317 aData.mToken = std::move(aToken);
318 aData.mStage = Stage::CreateDecoder;
319 RunStage(aData);
320 },
321 [&aData]() {
322 aData.mTokenRequest.Complete();
323 aData.mStage = Stage::None;
324 })
325 ->Track(aData.mTokenRequest);
326 aData.mStage = Stage::WaitForToken;
327 break;
328 }
329
330 case Stage::WaitForToken: {
331 MOZ_ASSERT(!aData.mToken);
332 MOZ_ASSERT(aData.mTokenRequest.Exists());
333 break;
334 }
335
336 case Stage::CreateDecoder: {
337 MOZ_ASSERT(aData.mToken);
338 MOZ_ASSERT(!aData.mDecoder);
339 MOZ_ASSERT(!aData.mInitRequest.Exists());
340
341 DoCreateDecoder(aData);
342 aData.mStage = Stage::WaitForInit;
343 break;
344 }
345
346 case Stage::WaitForInit: {
347 MOZ_ASSERT((aData.mDecoder && aData.mInitRequest.Exists()) ||
348 aData.mLiveToken);
349 break;
350 }
351 }
352 }
353
DoCreateDecoder(Data & aData)354 void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {
355 AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);
356 auto& ownerData = aData.mOwnerData;
357 auto& decoder = mOwner->GetDecoderData(aData.mTrack);
358
359 RefPtr<PDMFactory> platform = new PDMFactory();
360 if (decoder.IsEncrypted()) {
361 MOZ_ASSERT(mOwner->mCDMProxy);
362 platform->SetCDMProxy(mOwner->mCDMProxy);
363 }
364
365 RefPtr<PlatformDecoderModule::CreateDecoderPromise> p;
366 MediaFormatReader* owner = mOwner;
367 auto onWaitingForKeyEvent =
368 [owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)]() {
369 RefPtr<MediaFormatReader> mfr(owner);
370 MOZ_DIAGNOSTIC_ASSERT(mfr, "The MediaFormatReader didn't wait for us");
371 return mfr ? &mfr->OnTrackWaitingForKeyProducer() : nullptr;
372 };
373
374 switch (aData.mTrack) {
375 case TrackInfo::kAudioTrack: {
376 p = platform->CreateDecoder(
377 {*ownerData.GetCurrentInfo()->GetAsAudioInfo(), mOwner->mCrashHelper,
378 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
379 TrackInfo::kAudioTrack, std::move(onWaitingForKeyEvent)});
380 break;
381 }
382
383 case TrackType::kVideoTrack: {
384 // Decoders use the layers backend to decide if they can use hardware
385 // decoding, so specify LAYERS_NONE if we want to forcibly disable it.
386 using Option = CreateDecoderParams::Option;
387 using OptionSet = CreateDecoderParams::OptionSet;
388
389 p = platform->CreateDecoder(
390 {*ownerData.GetCurrentInfo()->GetAsVideoInfo(),
391 mOwner->mKnowsCompositor, mOwner->GetImageContainer(),
392 mOwner->mCrashHelper,
393 CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
394 TrackType::kVideoTrack, std::move(onWaitingForKeyEvent),
395 CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),
396 OptionSet(ownerData.mHardwareDecodingDisabled
397 ? Option::HardwareDecoderNotAllowed
398 : Option::Default)});
399 break;
400 }
401
402 default:
403 p = PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(
404 NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
405 }
406
407 aData.mLiveToken = MakeRefPtr<Data::DecoderCancelled>();
408
409 aData.mCreateDecoderPromise = p->Then(
410 mOwner->OwnerThread(), __func__,
411 [this, &aData, &ownerData, live = WeakPtr{aData.mLiveToken},
412 owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)](
413 RefPtr<MediaDataDecoder>&& aDecoder) {
414 if (!live) {
415 return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder),
416 __func__);
417 }
418 aData.mLiveToken = nullptr;
419 aData.mDecoder = new MediaDataDecoderProxy(
420 aDecoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));
421 aData.mDecoder = new AllocationWrapper(aData.mDecoder.forget(),
422 aData.mToken.forget());
423 DecoderDoctorLogger::LinkParentAndChild(
424 aData.mDecoder.get(), "decoder",
425 "MediaFormatReader::DecoderFactory", this);
426
427 DoInitDecoder(aData);
428
429 return CreateDecoderPromise::CreateAndResolve(aData.mDecoder, __func__);
430 },
431 [this, &aData,
432 live = WeakPtr{aData.mLiveToken}](const MediaResult& aError) {
433 NS_WARNING("Error constructing decoders");
434 if (!live) {
435 return CreateDecoderPromise::CreateAndReject(aError, __func__);
436 }
437 aData.mLiveToken = nullptr;
438 aData.mToken = nullptr;
439 aData.mStage = Stage::None;
440 aData.mOwnerData.mDescription = aError.Description();
441 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
442 "create_decoder_error", aError);
443 mOwner->NotifyError(aData.mTrack, aError);
444
445 return CreateDecoderPromise::CreateAndReject(aError, __func__);
446 });
447 }
448
DoInitDecoder(Data & aData)449 void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {
450 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);
451 auto& ownerData = aData.mOwnerData;
452
453 DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
454 "initialize_decoder", DDNoValue{});
455 aData.mDecoder->Init()
456 ->Then(
457 mOwner->OwnerThread(), __func__,
458 [this, &aData, &ownerData](TrackType aTrack) {
459 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
460 MEDIA_PLAYBACK);
461 aData.mInitRequest.Complete();
462 aData.mStage = Stage::None;
463 MutexAutoLock lock(ownerData.mMutex);
464 ownerData.mDecoder = std::move(aData.mDecoder);
465 ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
466 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
467 DDLogCategory::Log, "decoder_initialized", DDNoValue{});
468 DecoderDoctorLogger::LinkParentAndChild(
469 "MediaFormatReader::DecoderData", &ownerData, "decoder",
470 ownerData.mDecoder.get());
471 mOwner->SetVideoDecodeThreshold();
472 mOwner->ScheduleUpdate(aTrack);
473 if (aTrack == TrackInfo::kVideoTrack) {
474 DecoderBenchmark::CheckVersion(
475 ownerData.GetCurrentInfo()->mMimeType);
476 }
477 },
478 [this, &aData, &ownerData](const MediaResult& aError) {
479 AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
480 MEDIA_PLAYBACK);
481 aData.mInitRequest.Complete();
482 MOZ_RELEASE_ASSERT(!ownerData.mDecoder,
483 "Can't have a decoder already set");
484 aData.mStage = Stage::None;
485 mOwner->mShutdownPromisePool->ShutdownDecoder(
486 aData.mDecoder.forget());
487 DDLOGEX2("MediaFormatReader::DecoderFactory", this,
488 DDLogCategory::Log, "initialize_decoder_error", aError);
489 mOwner->NotifyError(aData.mTrack, aError);
490 })
491 ->Track(aData.mInitRequest);
492 }
493
494 // DemuxerProxy ensures that the original main demuxer is only ever accessed
495 // via its own dedicated task queue.
496 // This ensure that the reader's taskqueue will never blocked while a demuxer
497 // is itself blocked attempting to access the MediaCache or the MediaResource.
498 class MediaFormatReader::DemuxerProxy {
499 using TrackType = TrackInfo::TrackType;
500 class Wrapper;
501
502 public:
DemuxerProxy(MediaDataDemuxer * aDemuxer)503 explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)
504 : mTaskQueue(
505 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
506 "DemuxerProxy::mTaskQueue")),
507 mData(new Data(aDemuxer)) {
508 MOZ_COUNT_CTOR(DemuxerProxy);
509 }
510
MOZ_COUNTED_DTOR(DemuxerProxy)511 MOZ_COUNTED_DTOR(DemuxerProxy)
512
513 RefPtr<ShutdownPromise> Shutdown() {
514 RefPtr<Data> data = std::move(mData);
515 return InvokeAsync(mTaskQueue, __func__, [data]() {
516 // We need to clear our reference to the demuxer now. So that in the event
517 // the init promise wasn't resolved, such as what can happen with the
518 // mediasource demuxer that is waiting on more data, it will force the
519 // init promise to be rejected.
520 data->mDemuxer = nullptr;
521 data->mAudioDemuxer = nullptr;
522 data->mVideoDemuxer = nullptr;
523 return ShutdownPromise::CreateAndResolve(true, __func__);
524 });
525 }
526
527 RefPtr<MediaDataDemuxer::InitPromise> Init();
528
GetTrackDemuxer(TrackType aTrack,uint32_t aTrackNumber)529 Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {
530 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
531
532 switch (aTrack) {
533 case TrackInfo::kAudioTrack:
534 return mData->mAudioDemuxer;
535 case TrackInfo::kVideoTrack:
536 return mData->mVideoDemuxer;
537 default:
538 return nullptr;
539 }
540 }
541
GetNumberTracks(TrackType aTrack) const542 uint32_t GetNumberTracks(TrackType aTrack) const {
543 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
544
545 switch (aTrack) {
546 case TrackInfo::kAudioTrack:
547 return mData->mNumAudioTrack;
548 case TrackInfo::kVideoTrack:
549 return mData->mNumVideoTrack;
550 default:
551 return 0;
552 }
553 }
554
IsSeekable() const555 bool IsSeekable() const {
556 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
557
558 return mData->mSeekable;
559 }
560
IsSeekableOnlyInBufferedRanges() const561 bool IsSeekableOnlyInBufferedRanges() const {
562 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
563
564 return mData->mSeekableOnlyInBufferedRange;
565 }
566
GetCrypto() const567 UniquePtr<EncryptionInfo> GetCrypto() const {
568 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
569
570 if (!mData->mCrypto) {
571 return nullptr;
572 }
573 auto crypto = MakeUnique<EncryptionInfo>();
574 *crypto = *mData->mCrypto;
575 return crypto;
576 }
577
578 RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();
579
ShouldComputeStartTime() const580 bool ShouldComputeStartTime() const {
581 MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
582
583 return mData->mShouldComputeStartTime;
584 }
585
586 private:
587 const RefPtr<TaskQueue> mTaskQueue;
588 struct Data {
589 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)
590
Datamozilla::MediaFormatReader::DemuxerProxy::Data591 explicit Data(MediaDataDemuxer* aDemuxer)
592 : mInitDone(false), mDemuxer(aDemuxer) {}
593
594 Atomic<bool> mInitDone;
595 // Only ever accessed over mTaskQueue once.
596 RefPtr<MediaDataDemuxer> mDemuxer;
597 // Only accessed once InitPromise has been resolved and immutable after.
598 // So we can safely access them without the use of the mutex.
599 uint32_t mNumAudioTrack = 0;
600 RefPtr<Wrapper> mAudioDemuxer;
601 uint32_t mNumVideoTrack = 0;
602 RefPtr<Wrapper> mVideoDemuxer;
603 bool mSeekable = false;
604 bool mSeekableOnlyInBufferedRange = false;
605 bool mShouldComputeStartTime = true;
606 UniquePtr<EncryptionInfo> mCrypto;
607
608 private:
609 ~Data() = default;
610 };
611 RefPtr<Data> mData;
612 };
613
614 class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {
615 public:
Wrapper(MediaTrackDemuxer * aTrackDemuxer,TaskQueue * aTaskQueue)616 Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)
617 : mMutex("TrackDemuxer Mutex"),
618 mTaskQueue(aTaskQueue),
619 mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),
620 mInfo(aTrackDemuxer->GetInfo()),
621 mTrackDemuxer(aTrackDemuxer) {
622 DecoderDoctorLogger::LogConstructionAndBase(
623 "MediaFormatReader::DemuxerProxy::Wrapper", this,
624 static_cast<const MediaTrackDemuxer*>(this));
625 DecoderDoctorLogger::LinkParentAndChild(
626 "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
627 aTrackDemuxer);
628 }
629
GetInfo() const630 UniquePtr<TrackInfo> GetInfo() const override {
631 if (!mInfo) {
632 return nullptr;
633 }
634 return mInfo->Clone();
635 }
636
Seek(const TimeUnit & aTime)637 RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {
638 RefPtr<Wrapper> self = this;
639 return InvokeAsync(
640 mTaskQueue, __func__,
641 [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })
642 ->Then(
643 mTaskQueue, __func__,
644 [self](const TimeUnit& aTime) {
645 self->UpdateRandomAccessPoint();
646 return SeekPromise::CreateAndResolve(aTime, __func__);
647 },
648 [self](const MediaResult& aError) {
649 self->UpdateRandomAccessPoint();
650 return SeekPromise::CreateAndReject(aError, __func__);
651 });
652 }
653
GetSamples(int32_t aNumSamples)654 RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {
655 RefPtr<Wrapper> self = this;
656 return InvokeAsync(mTaskQueue, __func__,
657 [self, aNumSamples]() {
658 return self->mTrackDemuxer->GetSamples(aNumSamples);
659 })
660 ->Then(
661 mTaskQueue, __func__,
662 [self](RefPtr<SamplesHolder> aSamples) {
663 self->UpdateRandomAccessPoint();
664 return SamplesPromise::CreateAndResolve(aSamples.forget(),
665 __func__);
666 },
667 [self](const MediaResult& aError) {
668 self->UpdateRandomAccessPoint();
669 return SamplesPromise::CreateAndReject(aError, __func__);
670 });
671 }
672
GetSamplesMayBlock() const673 bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }
674
Reset()675 void Reset() override {
676 RefPtr<Wrapper> self = this;
677 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
678 "MediaFormatReader::DemuxerProxy::Wrapper::Reset",
679 [self]() { self->mTrackDemuxer->Reset(); }));
680 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
681 Unused << rv;
682 }
683
GetNextRandomAccessPoint(TimeUnit * aTime)684 nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {
685 MutexAutoLock lock(mMutex);
686 if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {
687 *aTime = mNextRandomAccessPoint;
688 }
689 return mNextRandomAccessPointResult;
690 }
691
SkipToNextRandomAccessPoint(const TimeUnit & aTimeThreshold)692 RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
693 const TimeUnit& aTimeThreshold) override {
694 RefPtr<Wrapper> self = this;
695 return InvokeAsync(
696 mTaskQueue, __func__,
697 [self, aTimeThreshold]() {
698 return self->mTrackDemuxer->SkipToNextRandomAccessPoint(
699 aTimeThreshold);
700 })
701 ->Then(
702 mTaskQueue, __func__,
703 [self](uint32_t aVal) {
704 self->UpdateRandomAccessPoint();
705 return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);
706 },
707 [self](const SkipFailureHolder& aError) {
708 self->UpdateRandomAccessPoint();
709 return SkipAccessPointPromise::CreateAndReject(aError, __func__);
710 });
711 }
712
GetBuffered()713 TimeIntervals GetBuffered() override {
714 MutexAutoLock lock(mMutex);
715 return mBuffered;
716 }
717
BreakCycles()718 void BreakCycles() override {}
719
720 private:
721 Mutex mMutex;
722 const RefPtr<TaskQueue> mTaskQueue;
723 const bool mGetSamplesMayBlock;
724 const UniquePtr<TrackInfo> mInfo;
725 // mTrackDemuxer is only ever accessed on demuxer's task queue.
726 RefPtr<MediaTrackDemuxer> mTrackDemuxer;
727 // All following members are protected by mMutex
728 nsresult mNextRandomAccessPointResult = NS_OK;
729 TimeUnit mNextRandomAccessPoint;
730 TimeIntervals mBuffered;
731 friend class DemuxerProxy;
732
~Wrapper()733 ~Wrapper() {
734 RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);
735 nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
736 "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
737 [trackDemuxer]() { trackDemuxer->BreakCycles(); }));
738 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
739 Unused << rv;
740 DecoderDoctorLogger::LogDestruction(
741 "MediaFormatReader::DemuxerProxy::Wrapper", this);
742 }
743
UpdateRandomAccessPoint()744 void UpdateRandomAccessPoint() {
745 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
746 if (!mTrackDemuxer) {
747 // Detached.
748 return;
749 }
750 MutexAutoLock lock(mMutex);
751 mNextRandomAccessPointResult =
752 mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);
753 }
754
UpdateBuffered()755 void UpdateBuffered() {
756 MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
757 if (!mTrackDemuxer) {
758 // Detached.
759 return;
760 }
761 MutexAutoLock lock(mMutex);
762 mBuffered = mTrackDemuxer->GetBuffered();
763 }
764 };
765
Init()766 RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {
767 AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);
768 using InitPromise = MediaDataDemuxer::InitPromise;
769
770 RefPtr<Data> data = mData;
771 RefPtr<TaskQueue> taskQueue = mTaskQueue;
772 return InvokeAsync(mTaskQueue, __func__,
773 [data, taskQueue]() {
774 if (!data->mDemuxer) {
775 return InitPromise::CreateAndReject(
776 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
777 }
778 return data->mDemuxer->Init();
779 })
780 ->Then(
781 taskQueue, __func__,
782 [data, taskQueue]() {
783 AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);
784 if (!data->mDemuxer) { // Was shutdown.
785 return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
786 __func__);
787 }
788 data->mNumAudioTrack =
789 data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
790 if (data->mNumAudioTrack) {
791 RefPtr<MediaTrackDemuxer> d =
792 data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
793 if (d) {
794 RefPtr<Wrapper> wrapper =
795 new DemuxerProxy::Wrapper(d, taskQueue);
796 wrapper->UpdateBuffered();
797 data->mAudioDemuxer = wrapper;
798 DecoderDoctorLogger::LinkParentAndChild(
799 data->mDemuxer.get(), "decoder factory wrapper",
800 "MediaFormatReader::DecoderFactory::Wrapper",
801 wrapper.get());
802 }
803 }
804 data->mNumVideoTrack =
805 data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
806 if (data->mNumVideoTrack) {
807 RefPtr<MediaTrackDemuxer> d =
808 data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
809 if (d) {
810 RefPtr<Wrapper> wrapper =
811 new DemuxerProxy::Wrapper(d, taskQueue);
812 wrapper->UpdateBuffered();
813 data->mVideoDemuxer = wrapper;
814 DecoderDoctorLogger::LinkParentAndChild(
815 data->mDemuxer.get(), "decoder factory wrapper",
816 "MediaFormatReader::DecoderFactory::Wrapper",
817 wrapper.get());
818 }
819 }
820 data->mCrypto = data->mDemuxer->GetCrypto();
821 data->mSeekable = data->mDemuxer->IsSeekable();
822 data->mSeekableOnlyInBufferedRange =
823 data->mDemuxer->IsSeekableOnlyInBufferedRanges();
824 data->mShouldComputeStartTime =
825 data->mDemuxer->ShouldComputeStartTime();
826 data->mInitDone = true;
827 return InitPromise::CreateAndResolve(NS_OK, __func__);
828 },
829 [](const MediaResult& aError) {
830 return InitPromise::CreateAndReject(aError, __func__);
831 });
832 }
833
834 RefPtr<MediaFormatReader::NotifyDataArrivedPromise>
NotifyDataArrived()835 MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
836 RefPtr<Data> data = mData;
837 return InvokeAsync(mTaskQueue, __func__, [data]() {
838 if (!data->mDemuxer) {
839 // Was shutdown.
840 return NotifyDataArrivedPromise::CreateAndReject(
841 NS_ERROR_DOM_MEDIA_CANCELED, __func__);
842 }
843 data->mDemuxer->NotifyDataArrived();
844 if (data->mAudioDemuxer) {
845 data->mAudioDemuxer->UpdateBuffered();
846 }
847 if (data->mVideoDemuxer) {
848 data->mVideoDemuxer->UpdateBuffered();
849 }
850 return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);
851 });
852 }
853
MediaFormatReader(MediaFormatReaderInit & aInit,MediaDataDemuxer * aDemuxer)854 MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
855 MediaDataDemuxer* aDemuxer)
856 : mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
857 "MediaFormatReader::mTaskQueue",
858 /* aSupportsTailDispatch = */ true)),
859 mAudio(this, MediaData::Type::AUDIO_DATA,
860 StaticPrefs::media_audio_max_decode_error()),
861 mVideo(this, MediaData::Type::VIDEO_DATA,
862 StaticPrefs::media_video_max_decode_error()),
863 mDemuxer(new DemuxerProxy(aDemuxer)),
864 mDemuxerInitDone(false),
865 mPendingNotifyDataArrived(false),
866 mLastReportedNumDecodedFrames(0),
867 mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),
868 mKnowsCompositor(aInit.mKnowsCompositor),
869 mInitDone(false),
870 mTrackDemuxersMayBlock(false),
871 mSeekScheduled(false),
872 mVideoFrameContainer(aInit.mVideoFrameContainer),
873 mCrashHelper(aInit.mCrashHelper),
874 mDecoderFactory(new DecoderFactory(this)),
875 mShutdownPromisePool(new ShutdownPromisePool()),
876 mBuffered(mTaskQueue, TimeIntervals(),
877 "MediaFormatReader::mBuffered (Canonical)"),
878 mFrameStats(aInit.mFrameStats),
879 mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID) {
880 MOZ_ASSERT(aDemuxer);
881 MOZ_COUNT_CTOR(MediaFormatReader);
882 DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
883 &mAudio);
884 DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
885 &mVideo);
886 DDLINKCHILD("demuxer", aDemuxer);
887 mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(
888 mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);
889 }
890
~MediaFormatReader()891 MediaFormatReader::~MediaFormatReader() {
892 MOZ_COUNT_DTOR(MediaFormatReader);
893 MOZ_ASSERT(mShutdown);
894 }
895
Shutdown()896 RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
897 MOZ_ASSERT(OnTaskQueue());
898 LOG("");
899
900 mDemuxerInitRequest.DisconnectIfExists();
901 mNotifyDataArrivedPromise.DisconnectIfExists();
902 mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
903 mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
904 mSkipRequest.DisconnectIfExists();
905 mSetCDMPromise.RejectIfExists(
906 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
907 "MediaFormatReader is shutting down"),
908 __func__);
909
910 if (mAudio.HasPromise()) {
911 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
912 }
913 if (mVideo.HasPromise()) {
914 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
915 }
916
917 if (HasAudio()) {
918 mAudio.ResetDemuxer();
919 mAudio.mTrackDemuxer->BreakCycles();
920 {
921 MutexAutoLock lock(mAudio.mMutex);
922 mAudio.mTrackDemuxer = nullptr;
923 }
924 mAudio.ResetState();
925 ShutdownDecoder(TrackInfo::kAudioTrack);
926 }
927
928 if (HasVideo()) {
929 mVideo.ResetDemuxer();
930 mVideo.mTrackDemuxer->BreakCycles();
931 {
932 MutexAutoLock lock(mVideo.mMutex);
933 mVideo.mTrackDemuxer = nullptr;
934 }
935 mVideo.ResetState();
936 ShutdownDecoder(TrackInfo::kVideoTrack);
937 }
938
939 mShutdownPromisePool->Track(mDemuxer->Shutdown());
940 mDemuxer = nullptr;
941
942 mOnTrackWaitingForKeyListener.Disconnect();
943
944 mShutdown = true;
945 return mShutdownPromisePool->Shutdown()->Then(
946 OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,
947 &MediaFormatReader::TearDownDecoders);
948 }
949
ShutdownDecoder(TrackType aTrack)950 void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {
951 LOGV("%s", TrackTypeToStr(aTrack));
952
953 // Shut down the pending decoder if any.
954 mDecoderFactory->ShutdownDecoder(aTrack);
955
956 auto& decoder = GetDecoderData(aTrack);
957 // Flush the decoder if necessary.
958 decoder.Flush();
959
960 // Shut down the decoder if any.
961 decoder.ShutdownDecoder();
962 }
963
NotifyDecoderBenchmarkStore()964 void MediaFormatReader::NotifyDecoderBenchmarkStore() {
965 MOZ_ASSERT(OnTaskQueue());
966 if (!StaticPrefs::media_mediacapabilities_from_database()) {
967 return;
968 }
969 auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);
970 if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {
971 VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());
972 info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));
973 mOnStoreDecoderBenchmark.Notify(std::move(info));
974 }
975 }
976
TearDownDecoders()977 RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
978 if (mAudio.mTaskQueue) {
979 mAudio.mTaskQueue->BeginShutdown();
980 mAudio.mTaskQueue->AwaitShutdownAndIdle();
981 mAudio.mTaskQueue = nullptr;
982 }
983 if (mVideo.mTaskQueue) {
984 mVideo.mTaskQueue->BeginShutdown();
985 mVideo.mTaskQueue->AwaitShutdownAndIdle();
986 mVideo.mTaskQueue = nullptr;
987 }
988
989 mDecoderFactory = nullptr;
990 mVideoFrameContainer = nullptr;
991
992 ReleaseResources();
993 mBuffered.DisconnectAll();
994 return mTaskQueue->BeginShutdown();
995 }
996
Init()997 nsresult MediaFormatReader::Init() {
998 MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
999
1000 mAudio.mTaskQueue =
1001 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1002 "MFR::mAudio::mTaskQueue");
1003
1004 mVideo.mTaskQueue =
1005 new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
1006 "MFR::mVideo::mTaskQueue");
1007
1008 return NS_OK;
1009 }
1010
ResolveSetCDMPromiseIfDone(TrackType aTrack)1011 bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {
1012 // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
1013 // and would create new one for specific track in the next Update.
1014 MOZ_ASSERT(OnTaskQueue());
1015
1016 if (mSetCDMPromise.IsEmpty()) {
1017 return true;
1018 }
1019
1020 MOZ_ASSERT(mCDMProxy);
1021 if (mSetCDMForTracks.contains(aTrack)) {
1022 mSetCDMForTracks -= aTrack;
1023 }
1024
1025 if (mSetCDMForTracks.isEmpty()) {
1026 LOGV("%s : Done ", __func__);
1027 mSetCDMPromise.Resolve(/* aIgnored = */ true, __func__);
1028 if (HasAudio()) {
1029 ScheduleUpdate(TrackInfo::kAudioTrack);
1030 }
1031 if (HasVideo()) {
1032 ScheduleUpdate(TrackInfo::kVideoTrack);
1033 }
1034 return true;
1035 }
1036 LOGV("%s : %s track is ready.", __func__, TrackTypeToStr(aTrack));
1037 return false;
1038 }
1039
PrepareToSetCDMForTrack(TrackType aTrack)1040 void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack) {
1041 MOZ_ASSERT(OnTaskQueue());
1042 LOGV("%s : %s", __func__, TrackTypeToStr(aTrack));
1043
1044 mSetCDMForTracks += aTrack;
1045 if (mCDMProxy) {
1046 // An old cdm proxy exists, so detaching old cdm proxy by shutting down
1047 // MediaDataDecoder.
1048 ShutdownDecoder(aTrack);
1049 }
1050 ScheduleUpdate(aTrack);
1051 }
1052
IsDecoderWaitingForCDM(TrackType aTrack)1053 bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack) {
1054 MOZ_ASSERT(OnTaskQueue());
1055 return GetDecoderData(aTrack).IsEncrypted() &&
1056 mSetCDMForTracks.contains(aTrack) && !mCDMProxy;
1057 }
1058
SetCDMProxy(CDMProxy * aProxy)1059 RefPtr<SetCDMPromise> MediaFormatReader::SetCDMProxy(CDMProxy* aProxy) {
1060 MOZ_ASSERT(OnTaskQueue());
1061 LOGV("SetCDMProxy (%p)", aProxy);
1062
1063 if (mShutdown) {
1064 return SetCDMPromise::CreateAndReject(
1065 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1066 "MediaFormatReader is shutting down"),
1067 __func__);
1068 }
1069
1070 mSetCDMPromise.RejectIfExists(
1071 MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
1072 "Another new CDM proxy is being set."),
1073 __func__);
1074
1075 // Shutdown all decoders as switching CDM proxy indicates that it's
1076 // inappropriate for the existing decoders to continue decoding via the old
1077 // CDM proxy.
1078 if (HasAudio()) {
1079 PrepareToSetCDMForTrack(TrackInfo::kAudioTrack);
1080 }
1081 if (HasVideo()) {
1082 PrepareToSetCDMForTrack(TrackInfo::kVideoTrack);
1083 }
1084
1085 mCDMProxy = aProxy;
1086
1087 if (!mInitDone || mSetCDMForTracks.isEmpty() || !mCDMProxy) {
1088 // 1) MFR is not initialized yet or
1089 // 2) Demuxer is initialized without active audio and video or
1090 // 3) A null cdm proxy is set
1091 // the promise can be resolved directly.
1092 mSetCDMForTracks.clear();
1093 return SetCDMPromise::CreateAndResolve(/* aIgnored = */ true, __func__);
1094 }
1095
1096 RefPtr<SetCDMPromise> p = mSetCDMPromise.Ensure(__func__);
1097 return p;
1098 }
1099
IsWaitingOnCDMResource()1100 bool MediaFormatReader::IsWaitingOnCDMResource() {
1101 MOZ_ASSERT(OnTaskQueue());
1102 return IsEncrypted() && !mCDMProxy;
1103 }
1104
1105 RefPtr<MediaFormatReader::MetadataPromise>
AsyncReadMetadata()1106 MediaFormatReader::AsyncReadMetadata() {
1107 AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK);
1108 MOZ_ASSERT(OnTaskQueue());
1109
1110 MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
1111
1112 if (mInitDone) {
1113 // We are returning from dormant.
1114 MetadataHolder metadata;
1115 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1116 return MetadataPromise::CreateAndResolve(std::move(metadata), __func__);
1117 }
1118
1119 RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
1120
1121 mDemuxer->Init()
1122 ->Then(OwnerThread(), __func__, this,
1123 &MediaFormatReader::OnDemuxerInitDone,
1124 &MediaFormatReader::OnDemuxerInitFailed)
1125 ->Track(mDemuxerInitRequest);
1126 return p;
1127 }
1128
OnDemuxerInitDone(const MediaResult & aResult)1129 void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
1130 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK);
1131 MOZ_ASSERT(OnTaskQueue());
1132 mDemuxerInitRequest.Complete();
1133
1134 if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1135 mMetadataPromise.Reject(aResult, __func__);
1136 return;
1137 }
1138
1139 mDemuxerInitDone = true;
1140
1141 UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
1142
1143 RefPtr<PDMFactory> platform;
1144 if (!IsWaitingOnCDMResource()) {
1145 platform = new PDMFactory();
1146 }
1147
1148 // To decode, we need valid video and a place to put it.
1149 bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
1150 GetImageContainer();
1151
1152 if (videoActive) {
1153 // We currently only handle the first video track.
1154 MutexAutoLock lock(mVideo.mMutex);
1155 mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1156 if (!mVideo.mTrackDemuxer) {
1157 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1158 return;
1159 }
1160
1161 UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
1162 videoActive = videoInfo && videoInfo->IsValid();
1163 if (videoActive) {
1164 if (platform && !platform->SupportsMimeType(videoInfo->mMimeType)) {
1165 // We have no decoder for this track. Error.
1166 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1167 return;
1168 }
1169 mInfo.mVideo = *videoInfo->GetAsVideoInfo();
1170 mVideo.mWorkingInfo = MakeUnique<VideoInfo>(mInfo.mVideo);
1171 for (const MetadataTag& tag : videoInfo->mTags) {
1172 tags->InsertOrUpdate(tag.mKey, tag.mValue);
1173 }
1174 mVideo.mOriginalInfo = std::move(videoInfo);
1175 mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
1176 } else {
1177 mVideo.mTrackDemuxer->BreakCycles();
1178 mVideo.mTrackDemuxer = nullptr;
1179 }
1180 }
1181
1182 bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1183 if (audioActive) {
1184 MutexAutoLock lock(mAudio.mMutex);
1185 mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1186 if (!mAudio.mTrackDemuxer) {
1187 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1188 return;
1189 }
1190
1191 UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
1192 // We actively ignore audio tracks that we know we can't play.
1193 audioActive =
1194 audioInfo && audioInfo->IsValid() &&
1195 (!platform || platform->SupportsMimeType(audioInfo->mMimeType));
1196
1197 if (audioActive) {
1198 mInfo.mAudio = *audioInfo->GetAsAudioInfo();
1199 mAudio.mWorkingInfo = MakeUnique<AudioInfo>(mInfo.mAudio);
1200 for (const MetadataTag& tag : audioInfo->mTags) {
1201 tags->InsertOrUpdate(tag.mKey, tag.mValue);
1202 }
1203 mAudio.mOriginalInfo = std::move(audioInfo);
1204 mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
1205 } else {
1206 mAudio.mTrackDemuxer->BreakCycles();
1207 mAudio.mTrackDemuxer = nullptr;
1208 }
1209 }
1210
1211 UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
1212 if (crypto && crypto->IsEncrypted()) {
1213 // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1214 for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
1215 mOnEncrypted.Notify(crypto->mInitDatas[i].mInitData,
1216 crypto->mInitDatas[i].mType);
1217 }
1218 mInfo.mCrypto = *crypto;
1219 }
1220
1221 auto videoDuration = HasVideo() ? mInfo.mVideo.mDuration : TimeUnit::Zero();
1222 auto audioDuration = HasAudio() ? mInfo.mAudio.mDuration : TimeUnit::Zero();
1223
1224 auto duration = std::max(videoDuration, audioDuration);
1225 if (duration.IsPositive()) {
1226 mInfo.mMetadataDuration = Some(duration);
1227 }
1228
1229 mInfo.mMediaSeekable = mDemuxer->IsSeekable();
1230 mInfo.mMediaSeekableOnlyInBufferedRanges =
1231 mDemuxer->IsSeekableOnlyInBufferedRanges();
1232
1233 if (!videoActive && !audioActive) {
1234 mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
1235 return;
1236 }
1237
1238 mTags = std::move(tags);
1239 mInitDone = true;
1240
1241 // Try to get the start time.
1242 // For MSE case, the start time of each track is assumed to be 0.
1243 // For others, we must demux the first sample to know the start time for each
1244 // track.
1245 if (!mDemuxer->ShouldComputeStartTime()) {
1246 mAudio.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1247 mVideo.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
1248 } else {
1249 if (HasAudio()) {
1250 RequestDemuxSamples(TrackInfo::kAudioTrack);
1251 }
1252
1253 if (HasVideo()) {
1254 RequestDemuxSamples(TrackInfo::kVideoTrack);
1255 }
1256 }
1257
1258 if (aResult != NS_OK) {
1259 mOnDecodeWarning.Notify(aResult);
1260 }
1261
1262 MaybeResolveMetadataPromise();
1263 }
1264
MaybeResolveMetadataPromise()1265 void MediaFormatReader::MaybeResolveMetadataPromise() {
1266 MOZ_ASSERT(OnTaskQueue());
1267
1268 if ((HasAudio() && mAudio.mFirstDemuxedSampleTime.isNothing()) ||
1269 (HasVideo() && mVideo.mFirstDemuxedSampleTime.isNothing())) {
1270 return;
1271 }
1272
1273 TimeUnit startTime =
1274 std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
1275 mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
1276
1277 if (!startTime.IsInfinite()) {
1278 mInfo.mStartTime = startTime; // mInfo.mStartTime is initialized to 0.
1279 }
1280
1281 MetadataHolder metadata;
1282 metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
1283 metadata.mTags = mTags->Count() ? std::move(mTags) : nullptr;
1284
1285 // We now have all the informations required to calculate the initial buffered
1286 // range.
1287 mHasStartTime = true;
1288 UpdateBuffered();
1289
1290 mMetadataPromise.Resolve(std::move(metadata), __func__);
1291 }
1292
IsEncrypted() const1293 bool MediaFormatReader::IsEncrypted() const {
1294 return (HasAudio() && mAudio.GetCurrentInfo()->mCrypto.IsEncrypted()) ||
1295 (HasVideo() && mVideo.GetCurrentInfo()->mCrypto.IsEncrypted());
1296 }
1297
OnDemuxerInitFailed(const MediaResult & aError)1298 void MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError) {
1299 mDemuxerInitRequest.Complete();
1300 mMetadataPromise.Reject(aError, __func__);
1301 }
1302
ReadUpdatedMetadata(MediaInfo * aInfo)1303 void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) {
1304 // Called on the MDSM's TaskQueue.
1305 {
1306 MutexAutoLock lock(mVideo.mMutex);
1307 if (HasVideo()) {
1308 aInfo->mVideo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
1309 }
1310 }
1311 {
1312 MutexAutoLock lock(mAudio.mMutex);
1313 if (HasAudio()) {
1314 aInfo->mAudio = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
1315 }
1316 }
1317 }
1318
GetDecoderData(TrackType aTrack)1319 MediaFormatReader::DecoderData& MediaFormatReader::GetDecoderData(
1320 TrackType aTrack) {
1321 MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
1322 aTrack == TrackInfo::kVideoTrack);
1323 if (aTrack == TrackInfo::kAudioTrack) {
1324 return mAudio;
1325 }
1326 return mVideo;
1327 }
1328
ShouldSkip(TimeUnit aTimeThreshold,bool aRequestNextVideoKeyFrame)1329 Maybe<TimeUnit> MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold,
1330 bool aRequestNextVideoKeyFrame) {
1331 MOZ_ASSERT(OnTaskQueue());
1332 MOZ_ASSERT(HasVideo());
1333
1334 if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
1335 return Nothing();
1336 }
1337
1338 // Ensure we have no pending seek going as skip-to-keyframe could return out
1339 // of date information.
1340 if (mVideo.HasInternalSeekPending()) {
1341 return Nothing();
1342 }
1343
1344 TimeUnit nextKeyframe;
1345 nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
1346 if (NS_FAILED(rv)) {
1347 // Only OggTrackDemuxer with video type gets into here.
1348 // We don't support skip-to-next-frame for this case.
1349 return Nothing();
1350 }
1351
1352 const bool isNextKeyframeValid =
1353 nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
1354 // If we request the next keyframe, only return times greater than
1355 // aTimeThreshold. Otherwise, data will be already behind the threshold and
1356 // will be eventually discarded somewhere in the media pipeline.
1357 if (aRequestNextVideoKeyFrame && isNextKeyframeValid &&
1358 nextKeyframe > aTimeThreshold) {
1359 return Some(nextKeyframe);
1360 }
1361
1362 const bool isNextVideoBehindTheThreshold =
1363 (isNextKeyframeValid && nextKeyframe <= aTimeThreshold) ||
1364 GetInternalSeekTargetEndTime() < aTimeThreshold;
1365 return isNextVideoBehindTheThreshold ? Some(aTimeThreshold) : Nothing();
1366 }
1367
RequestVideoData(const TimeUnit & aTimeThreshold,bool aRequestNextVideoKeyFrame)1368 RefPtr<MediaFormatReader::VideoDataPromise> MediaFormatReader::RequestVideoData(
1369 const TimeUnit& aTimeThreshold, bool aRequestNextVideoKeyFrame) {
1370 MOZ_ASSERT(OnTaskQueue());
1371 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
1372 "No sample requests allowed while seeking");
1373 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
1374 MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
1375 mVideo.mTimeThreshold.isSome());
1376 MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
1377 LOGV("RequestVideoData(%" PRId64 "), requestNextKeyFrame=%d",
1378 aTimeThreshold.ToMicroseconds(), aRequestNextVideoKeyFrame);
1379
1380 if (!HasVideo()) {
1381 LOG("called with no video track");
1382 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1383 __func__);
1384 }
1385
1386 if (IsSeeking()) {
1387 LOG("called mid-seek. Rejecting.");
1388 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1389 __func__);
1390 }
1391
1392 if (mShutdown) {
1393 NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
1394 return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1395 __func__);
1396 }
1397
1398 if (Maybe<TimeUnit> target =
1399 ShouldSkip(aTimeThreshold, aRequestNextVideoKeyFrame)) {
1400 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1401 SkipVideoDemuxToNextKeyFrame(*target);
1402 return p;
1403 }
1404
1405 RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
1406 ScheduleUpdate(TrackInfo::kVideoTrack);
1407
1408 return p;
1409 }
1410
OnDemuxFailed(TrackType aTrack,const MediaResult & aError)1411 void MediaFormatReader::OnDemuxFailed(TrackType aTrack,
1412 const MediaResult& aError) {
1413 AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK);
1414 MOZ_ASSERT(OnTaskQueue());
1415 LOG("Failed to demux %s, failure:%s",
1416 aTrack == TrackType::kVideoTrack ? "video" : "audio",
1417 aError.ErrorName().get());
1418 auto& decoder = GetDecoderData(aTrack);
1419 decoder.mDemuxRequest.Complete();
1420 switch (aError.Code()) {
1421 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1422 DDLOG(DDLogCategory::Log,
1423 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1424 : "audio_demux_interruption",
1425 aError);
1426 if (!decoder.mWaitingForData) {
1427 decoder.RequestDrain();
1428 }
1429 NotifyEndOfStream(aTrack);
1430 break;
1431 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1432 DDLOG(DDLogCategory::Log,
1433 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1434 : "audio_demux_interruption",
1435 aError);
1436 if (!decoder.mWaitingForData) {
1437 decoder.RequestDrain();
1438 }
1439 NotifyWaitingForData(aTrack);
1440 break;
1441 case NS_ERROR_DOM_MEDIA_CANCELED:
1442 DDLOG(DDLogCategory::Log,
1443 aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
1444 : "audio_demux_interruption",
1445 aError);
1446 if (decoder.HasPromise()) {
1447 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
1448 }
1449 break;
1450 default:
1451 DDLOG(DDLogCategory::Log,
1452 aTrack == TrackType::kVideoTrack ? "video_demux_error"
1453 : "audio_demux_error",
1454 aError);
1455 NotifyError(aTrack, aError);
1456 break;
1457 }
1458 }
1459
DoDemuxVideo()1460 void MediaFormatReader::DoDemuxVideo() {
1461 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK);
1462 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1463
1464 DDLOG(DDLogCategory::Log, "video_demuxing", DDNoValue{});
1465 PerformanceRecorder perfRecorder(
1466 PerformanceRecorder::Stage::RequestDemux,
1467 mVideo.GetCurrentInfo()->GetAsVideoInfo()->mImage.height);
1468 perfRecorder.Start();
1469 auto p = mVideo.mTrackDemuxer->GetSamples(1);
1470
1471 RefPtr<MediaFormatReader> self = this;
1472 if (mVideo.mFirstDemuxedSampleTime.isNothing()) {
1473 p = p->Then(
1474 OwnerThread(), __func__,
1475 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1476 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
1477 MEDIA_PLAYBACK);
1478 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxed",
1479 DDNoValue{});
1480 self->OnFirstDemuxCompleted(TrackInfo::kVideoTrack, aSamples);
1481 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1482 },
1483 [self](const MediaResult& aError) {
1484 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
1485 MEDIA_PLAYBACK);
1486 DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxing_error",
1487 aError);
1488 self->OnFirstDemuxFailed(TrackInfo::kVideoTrack, aError);
1489 return SamplesPromise::CreateAndReject(aError, __func__);
1490 });
1491 }
1492
1493 p->Then(
1494 OwnerThread(), __func__,
1495 [self, perfRecorder(std::move(perfRecorder))](
1496 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) mutable {
1497 perfRecorder.End();
1498 self->OnVideoDemuxCompleted(std::move(aSamples));
1499 },
1500 [self](const MediaResult& aError) { self->OnVideoDemuxFailed(aError); })
1501 ->Track(mVideo.mDemuxRequest);
1502 }
1503
OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)1504 void MediaFormatReader::OnVideoDemuxCompleted(
1505 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1506 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
1507 MEDIA_PLAYBACK);
1508 LOGV("%zu video samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1509 aSamples->GetSamples()[0]->mTrackInfo
1510 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1511 : 0);
1512 DDLOG(DDLogCategory::Log, "video_demuxed_samples",
1513 uint64_t(aSamples->GetSamples().Length()));
1514 mVideo.mDemuxRequest.Complete();
1515 mVideo.mQueuedSamples.AppendElements(aSamples->GetSamples());
1516 ScheduleUpdate(TrackInfo::kVideoTrack);
1517 }
1518
1519 RefPtr<MediaFormatReader::AudioDataPromise>
RequestAudioData()1520 MediaFormatReader::RequestAudioData() {
1521 MOZ_ASSERT(OnTaskQueue());
1522 MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
1523 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || mSeekPromise.IsEmpty(),
1524 "No sample requests allowed while seeking");
1525 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !mAudio.mSeekRequest.Exists() ||
1526 mAudio.mTimeThreshold.isSome());
1527 MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !IsSeeking(), "called mid-seek");
1528 LOGV("");
1529
1530 if (!HasAudio()) {
1531 LOG("called with no audio track");
1532 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
1533 __func__);
1534 }
1535
1536 if (IsSeeking()) {
1537 LOG("called mid-seek. Rejecting.");
1538 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1539 __func__);
1540 }
1541
1542 if (mShutdown) {
1543 NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
1544 return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
1545 __func__);
1546 }
1547
1548 RefPtr<AudioDataPromise> p = mAudio.EnsurePromise(__func__);
1549 ScheduleUpdate(TrackInfo::kAudioTrack);
1550
1551 return p;
1552 }
1553
DoDemuxAudio()1554 void MediaFormatReader::DoDemuxAudio() {
1555 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK);
1556 using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
1557
1558 DDLOG(DDLogCategory::Log, "audio_demuxing", DDNoValue{});
1559 PerformanceRecorder perfRecorder(PerformanceRecorder::Stage::RequestDemux);
1560 perfRecorder.Start();
1561 auto p = mAudio.mTrackDemuxer->GetSamples(1);
1562
1563 RefPtr<MediaFormatReader> self = this;
1564 if (mAudio.mFirstDemuxedSampleTime.isNothing()) {
1565 p = p->Then(
1566 OwnerThread(), __func__,
1567 [self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1568 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
1569 MEDIA_PLAYBACK);
1570 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxed",
1571 DDNoValue{});
1572 self->OnFirstDemuxCompleted(TrackInfo::kAudioTrack, aSamples);
1573 return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
1574 },
1575 [self](const MediaResult& aError) {
1576 AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
1577 MEDIA_PLAYBACK);
1578 DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxing_error",
1579 aError);
1580 self->OnFirstDemuxFailed(TrackInfo::kAudioTrack, aError);
1581 return SamplesPromise::CreateAndReject(aError, __func__);
1582 });
1583 }
1584
1585 p->Then(
1586 OwnerThread(), __func__,
1587 [self, perfRecorder(std::move(perfRecorder))](
1588 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) mutable {
1589 perfRecorder.End();
1590 self->OnAudioDemuxCompleted(std::move(aSamples));
1591 },
1592 [self](const MediaResult& aError) { self->OnAudioDemuxFailed(aError); })
1593 ->Track(mAudio.mDemuxRequest);
1594 }
1595
OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)1596 void MediaFormatReader::OnAudioDemuxCompleted(
1597 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1598 LOGV("%zu audio samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
1599 aSamples->GetSamples()[0]->mTrackInfo
1600 ? aSamples->GetSamples()[0]->mTrackInfo->GetID()
1601 : 0);
1602 DDLOG(DDLogCategory::Log, "audio_demuxed_samples",
1603 uint64_t(aSamples->GetSamples().Length()));
1604 mAudio.mDemuxRequest.Complete();
1605 mAudio.mQueuedSamples.AppendElements(aSamples->GetSamples());
1606 ScheduleUpdate(TrackInfo::kAudioTrack);
1607 }
1608
NotifyNewOutput(TrackType aTrack,MediaDataDecoder::DecodedData && aResults)1609 void MediaFormatReader::NotifyNewOutput(
1610 TrackType aTrack, MediaDataDecoder::DecodedData&& aResults) {
1611 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK);
1612 MOZ_ASSERT(OnTaskQueue());
1613 auto& decoder = GetDecoderData(aTrack);
1614 if (aResults.IsEmpty()) {
1615 DDLOG(DDLogCategory::Log,
1616 aTrack == TrackInfo::kAudioTrack ? "decoded_audio" : "decoded_video",
1617 "no output samples");
1618 } else
1619 for (auto&& sample : aResults) {
1620 if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
1621 switch (sample->mType) {
1622 case MediaData::Type::AUDIO_DATA:
1623 DDLOGPR(DDLogCategory::Log,
1624 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1625 : "decoded_got_audio!?",
1626 "{\"type\":\"AudioData\", \"offset\":%" PRIi64
1627 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1628 ", \"duration_us\":%" PRIi64 ", \"frames\":%" PRIu32
1629 ", \"channels\":%" PRIu32 ", \"rate\":%" PRIu32
1630 ", \"bytes\":%zu}",
1631 sample->mOffset, sample->mTime.ToMicroseconds(),
1632 sample->mTimecode.ToMicroseconds(),
1633 sample->mDuration.ToMicroseconds(),
1634 sample->As<AudioData>()->Frames(),
1635 sample->As<AudioData>()->mChannels,
1636 sample->As<AudioData>()->mRate,
1637 sample->As<AudioData>()->Data().Length());
1638 break;
1639 case MediaData::Type::VIDEO_DATA:
1640 DDLOGPR(DDLogCategory::Log,
1641 aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1642 : "decoded_got_video!?",
1643 "{\"type\":\"VideoData\", \"offset\":%" PRIi64
1644 ", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1645 ", \"duration_us\":%" PRIi64
1646 ", \"kf\":%s, \"size\":[%" PRIi32 ",%" PRIi32 "]}",
1647 sample->mOffset, sample->mTime.ToMicroseconds(),
1648 sample->mTimecode.ToMicroseconds(),
1649 sample->mDuration.ToMicroseconds(),
1650 sample->mKeyframe ? "true" : "false",
1651 sample->As<VideoData>()->mDisplay.width,
1652 sample->As<VideoData>()->mDisplay.height);
1653 break;
1654 case MediaData::Type::RAW_DATA:
1655 DDLOGPR(DDLogCategory::Log,
1656 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1657 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1658 : "decoded_?",
1659 "{\"type\":\"RawData\", \"offset\":%" PRIi64
1660 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1661 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1662 sample->mOffset, sample->mTime.ToMicroseconds(),
1663 sample->mTimecode.ToMicroseconds(),
1664 sample->mDuration.ToMicroseconds(),
1665 sample->mKeyframe ? "true" : "false");
1666 break;
1667 case MediaData::Type::NULL_DATA:
1668 DDLOGPR(DDLogCategory::Log,
1669 aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
1670 : aTrack == TrackInfo::kVideoTrack ? "decoded_video"
1671 : "decoded_?",
1672 "{\"type\":\"NullData\", \"offset\":%" PRIi64
1673 " \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1674 ", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
1675 sample->mOffset, sample->mTime.ToMicroseconds(),
1676 sample->mTimecode.ToMicroseconds(),
1677 sample->mDuration.ToMicroseconds(),
1678 sample->mKeyframe ? "true" : "false");
1679 break;
1680 }
1681 }
1682 LOGV("Received new %s sample time:%" PRId64 " duration:%" PRId64,
1683 TrackTypeToStr(aTrack), sample->mTime.ToMicroseconds(),
1684 sample->mDuration.ToMicroseconds());
1685 decoder.mOutput.AppendElement(sample);
1686 decoder.mNumSamplesOutput++;
1687 decoder.mNumOfConsecutiveError = 0;
1688 }
1689 LOG("Done processing new %s samples", TrackTypeToStr(aTrack));
1690
1691 if (!aResults.IsEmpty()) {
1692 // We have decoded our first frame, we can now starts to skip future errors.
1693 decoder.mFirstFrameTime.reset();
1694 }
1695 ScheduleUpdate(aTrack);
1696 }
1697
NotifyError(TrackType aTrack,const MediaResult & aError)1698 void MediaFormatReader::NotifyError(TrackType aTrack,
1699 const MediaResult& aError) {
1700 MOZ_ASSERT(OnTaskQueue());
1701 NS_WARNING(aError.Description().get());
1702 LOGV("%s Decoding error", TrackTypeToStr(aTrack));
1703 auto& decoder = GetDecoderData(aTrack);
1704 decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
1705
1706 ScheduleUpdate(aTrack);
1707 }
1708
NotifyWaitingForData(TrackType aTrack)1709 void MediaFormatReader::NotifyWaitingForData(TrackType aTrack) {
1710 MOZ_ASSERT(OnTaskQueue());
1711 auto& decoder = GetDecoderData(aTrack);
1712 decoder.mWaitingForData = true;
1713 if (decoder.mTimeThreshold) {
1714 decoder.mTimeThreshold.ref().mWaiting = true;
1715 }
1716 ScheduleUpdate(aTrack);
1717 }
1718
NotifyWaitingForKey(TrackType aTrack)1719 void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack) {
1720 MOZ_ASSERT(OnTaskQueue());
1721 auto& decoder = GetDecoderData(aTrack);
1722 mOnWaitingForKey.Notify();
1723 if (!decoder.mDecodeRequest.Exists()) {
1724 LOGV("WaitingForKey received while no pending decode. Ignoring");
1725 return;
1726 }
1727 decoder.mWaitingForKey = true;
1728 ScheduleUpdate(aTrack);
1729 }
1730
NotifyEndOfStream(TrackType aTrack)1731 void MediaFormatReader::NotifyEndOfStream(TrackType aTrack) {
1732 MOZ_ASSERT(OnTaskQueue());
1733 auto& decoder = GetDecoderData(aTrack);
1734 decoder.mDemuxEOS = true;
1735 ScheduleUpdate(aTrack);
1736 }
1737
NeedInput(DecoderData & aDecoder)1738 bool MediaFormatReader::NeedInput(DecoderData& aDecoder) {
1739 // The decoder will not be fed a new raw sample until the current decoding
1740 // requests has completed.
1741 return (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
1742 !aDecoder.HasPendingDrain() && !aDecoder.HasFatalError() &&
1743 !aDecoder.mDemuxRequest.Exists() && !aDecoder.mOutput.Length() &&
1744 !aDecoder.HasInternalSeekPending() &&
1745 !aDecoder.mDecodeRequest.Exists();
1746 }
1747
ScheduleUpdate(TrackType aTrack)1748 void MediaFormatReader::ScheduleUpdate(TrackType aTrack) {
1749 MOZ_ASSERT(OnTaskQueue());
1750 if (mShutdown) {
1751 return;
1752 }
1753 auto& decoder = GetDecoderData(aTrack);
1754 MOZ_RELEASE_ASSERT(decoder.GetCurrentInfo(),
1755 "Can only schedule update when track exists");
1756
1757 if (decoder.mUpdateScheduled) {
1758 return;
1759 }
1760 LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
1761 decoder.mUpdateScheduled = true;
1762 RefPtr<nsIRunnable> task(NewRunnableMethod<TrackType>(
1763 "MediaFormatReader::Update", this, &MediaFormatReader::Update, aTrack));
1764 nsresult rv = OwnerThread()->Dispatch(task.forget());
1765 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1766 Unused << rv;
1767 }
1768
UpdateReceivedNewData(TrackType aTrack)1769 bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) {
1770 MOZ_ASSERT(OnTaskQueue());
1771 auto& decoder = GetDecoderData(aTrack);
1772
1773 if (!decoder.mReceivedNewData) {
1774 return false;
1775 }
1776
1777 // We do not want to clear mWaitingForData while there are pending
1778 // demuxing or seeking operations that could affect the value of this flag.
1779 // This is in order to ensure that we will retry once they complete as we may
1780 // now have new data that could potentially allow those operations to
1781 // successfully complete if tried again.
1782 if (decoder.mSeekRequest.Exists()) {
1783 // Nothing more to do until this operation complete.
1784 return true;
1785 }
1786
1787 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
1788 LOGV("Skipping in progress, nothing more to do");
1789 return true;
1790 }
1791
1792 if (decoder.mDemuxRequest.Exists()) {
1793 // We may have pending operations to process, so we want to continue
1794 // after UpdateReceivedNewData returns.
1795 return false;
1796 }
1797
1798 if (decoder.HasPendingDrain()) {
1799 // We do not want to clear mWaitingForData or mDemuxEOS while
1800 // a drain is in progress in order to properly complete the operation.
1801 return false;
1802 }
1803
1804 decoder.mReceivedNewData = false;
1805 if (decoder.mTimeThreshold) {
1806 decoder.mTimeThreshold.ref().mWaiting = false;
1807 }
1808 decoder.mWaitingForData = false;
1809
1810 if (decoder.HasFatalError()) {
1811 return false;
1812 }
1813
1814 if (!mSeekPromise.IsEmpty() &&
1815 (!IsVideoSeeking() || aTrack == TrackInfo::kVideoTrack)) {
1816 MOZ_ASSERT(!decoder.HasPromise());
1817 MOZ_DIAGNOSTIC_ASSERT(
1818 (IsVideoSeeking() || !mAudio.mTimeThreshold) && !mVideo.mTimeThreshold,
1819 "InternalSeek must have been aborted when Seek was first called");
1820 MOZ_DIAGNOSTIC_ASSERT(
1821 (IsVideoSeeking() || !mAudio.HasWaitingPromise()) &&
1822 !mVideo.HasWaitingPromise(),
1823 "Waiting promises must have been rejected when Seek was first called");
1824 if (mVideo.mSeekRequest.Exists() ||
1825 (!IsVideoSeeking() && mAudio.mSeekRequest.Exists())) {
1826 // Already waiting for a seek to complete. Nothing more to do.
1827 return true;
1828 }
1829 LOG("Attempting Seek");
1830 ScheduleSeek();
1831 return true;
1832 }
1833 if (decoder.HasInternalSeekPending() || decoder.HasWaitingPromise()) {
1834 if (decoder.HasInternalSeekPending()) {
1835 LOG("Attempting Internal Seek");
1836 InternalSeek(aTrack, decoder.mTimeThreshold.ref());
1837 }
1838 if (decoder.HasWaitingPromise() && !decoder.IsWaitingForKey() &&
1839 !decoder.IsWaitingForData()) {
1840 MOZ_ASSERT(!decoder.HasPromise());
1841 LOG("We have new data. Resolving WaitingPromise");
1842 decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
1843 }
1844 return true;
1845 }
1846 return false;
1847 }
1848
RequestDemuxSamples(TrackType aTrack)1849 void MediaFormatReader::RequestDemuxSamples(TrackType aTrack) {
1850 MOZ_ASSERT(OnTaskQueue());
1851 auto& decoder = GetDecoderData(aTrack);
1852 MOZ_ASSERT(!decoder.mDemuxRequest.Exists());
1853
1854 if (!decoder.mQueuedSamples.IsEmpty()) {
1855 // No need to demux new samples.
1856 return;
1857 }
1858
1859 if (decoder.mDemuxEOS) {
1860 // Nothing left to demux.
1861 // We do not want to attempt to demux while in waiting for data mode
1862 // as it would retrigger an unnecessary drain.
1863 return;
1864 }
1865
1866 LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
1867 if (aTrack == TrackInfo::kVideoTrack) {
1868 DoDemuxVideo();
1869 } else {
1870 DoDemuxAudio();
1871 }
1872 }
1873
DecodeDemuxedSamples(TrackType aTrack,MediaRawData * aSample)1874 void MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
1875 MediaRawData* aSample) {
1876 MOZ_ASSERT(OnTaskQueue());
1877 auto& decoder = GetDecoderData(aTrack);
1878 RefPtr<MediaFormatReader> self = this;
1879 decoder.mFlushed = false;
1880 DDLOGPR(DDLogCategory::Log,
1881 aTrack == TrackInfo::kAudioTrack ? "decode_audio"
1882 : aTrack == TrackInfo::kVideoTrack ? "decode_video"
1883 : "decode_?",
1884 "{\"type\":\"MediaRawData\", \"offset\":%" PRIi64
1885 ", \"bytes\":%zu, \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
1886 ", \"duration_us\":%" PRIi64 ",%s%s}",
1887 aSample->mOffset, aSample->Size(), aSample->mTime.ToMicroseconds(),
1888 aSample->mTimecode.ToMicroseconds(),
1889 aSample->mDuration.ToMicroseconds(), aSample->mKeyframe ? " kf" : "",
1890 aSample->mEOS ? " eos" : "");
1891
1892 const int32_t height =
1893 aTrack == TrackInfo::kVideoTrack
1894 ? decoder.GetCurrentInfo()->GetAsVideoInfo()->mImage.height
1895 : 0;
1896 MediaInfoFlag flag = MediaInfoFlag::None;
1897 flag |=
1898 aSample->mKeyframe ? MediaInfoFlag::KeyFrame : MediaInfoFlag::NonKeyFrame;
1899 if (aTrack == TrackInfo::kVideoTrack) {
1900 flag |= VideoIsHardwareAccelerated() ? MediaInfoFlag::HardwareDecoding
1901 : MediaInfoFlag::SoftwareDecoding;
1902 const nsCString& mimeType = decoder.GetCurrentInfo()->mMimeType;
1903 if (MP4Decoder::IsH264(mimeType)) {
1904 flag |= MediaInfoFlag::VIDEO_H264;
1905 } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP8)) {
1906 flag |= MediaInfoFlag::VIDEO_VP8;
1907 } else if (VPXDecoder::IsVPX(mimeType, VPXDecoder::VP9)) {
1908 flag |= MediaInfoFlag::VIDEO_VP9;
1909 }
1910 #ifdef MOZ_AV1
1911 else if (AOMDecoder::IsAV1(mimeType)) {
1912 flag |= MediaInfoFlag::VIDEO_AV1;
1913 }
1914 #endif
1915 }
1916 PerformanceRecorder perfRecorder(PerformanceRecorder::Stage::RequestDecode,
1917 height, flag);
1918 perfRecorder.Start();
1919 decoder.mDecoder->Decode(aSample)
1920 ->Then(
1921 mTaskQueue, __func__,
1922 [self, aTrack, &decoder, perfRecorder(std::move(perfRecorder))](
1923 MediaDataDecoder::DecodedData&& aResults) mutable {
1924 perfRecorder.End();
1925 decoder.mDecodeRequest.Complete();
1926 self->NotifyNewOutput(aTrack, std::move(aResults));
1927 },
1928 [self, aTrack, &decoder](const MediaResult& aError) {
1929 decoder.mDecodeRequest.Complete();
1930 self->NotifyError(aTrack, aError);
1931 })
1932 ->Track(decoder.mDecodeRequest);
1933 }
1934
HandleDemuxedSamples(TrackType aTrack,FrameStatistics::AutoNotifyDecoded & aA)1935 void MediaFormatReader::HandleDemuxedSamples(
1936 TrackType aTrack, FrameStatistics::AutoNotifyDecoded& aA) {
1937 MOZ_ASSERT(OnTaskQueue());
1938
1939 auto& decoder = GetDecoderData(aTrack);
1940
1941 if (decoder.mFlushing) {
1942 LOGV("Decoder operation in progress, let it complete.");
1943 return;
1944 }
1945
1946 if (decoder.mQueuedSamples.IsEmpty()) {
1947 return;
1948 }
1949
1950 RefPtr<MediaRawData> sample = decoder.mQueuedSamples[0];
1951 const RefPtr<TrackInfoSharedPtr> info = sample->mTrackInfo;
1952
1953 if (info && decoder.mLastStreamSourceID != info->GetID()) {
1954 nsTArray<RefPtr<MediaRawData>> samples;
1955 if (decoder.mDecoder) {
1956 bool recyclable =
1957 StaticPrefs::media_decoder_recycle_enabled() &&
1958 decoder.mDecoder->SupportDecoderRecycling() &&
1959 (*info)->mCrypto.mCryptoScheme ==
1960 decoder.GetCurrentInfo()->mCrypto.mCryptoScheme &&
1961 (*info)->mMimeType == decoder.GetCurrentInfo()->mMimeType;
1962 if (!recyclable && decoder.mTimeThreshold.isNothing() &&
1963 (decoder.mNextStreamSourceID.isNothing() ||
1964 decoder.mNextStreamSourceID.ref() != info->GetID())) {
1965 LOG("%s stream id has changed from:%d to:%d, draining decoder.",
1966 TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
1967 decoder.RequestDrain();
1968 decoder.mNextStreamSourceID = Some(info->GetID());
1969 ScheduleUpdate(aTrack);
1970 return;
1971 }
1972
1973 // If flushing is required, it will clear our array of queued samples.
1974 // So we may need to make a copy.
1975 samples = decoder.mQueuedSamples.Clone();
1976 if (!recyclable) {
1977 LOG("Decoder does not support recycling, recreate decoder.");
1978 ShutdownDecoder(aTrack);
1979 // We're going to be using a new decoder following the change of content
1980 // We can attempt to use hardware decoding again.
1981 decoder.mHardwareDecodingDisabled = false;
1982 } else if (decoder.HasWaitingPromise()) {
1983 decoder.Flush();
1984 }
1985 }
1986
1987 nsPrintfCString markerString(
1988 "%s stream id changed from:%" PRIu32 " to:%" PRIu32,
1989 TrackTypeToStr(aTrack), decoder.mLastStreamSourceID, info->GetID());
1990 PROFILER_MARKER_TEXT("StreamID Change", MEDIA_PLAYBACK, {}, markerString);
1991 LOG("%s", markerString.get());
1992
1993 if (aTrack == TrackInfo::kVideoTrack) {
1994 // We are about to create a new decoder thus the benchmark,
1995 // up to this point, is stored.
1996 NotifyDecoderBenchmarkStore();
1997 }
1998 decoder.mNextStreamSourceID.reset();
1999 decoder.mLastStreamSourceID = info->GetID();
2000 decoder.mInfo = info;
2001 {
2002 MutexAutoLock lock(decoder.mMutex);
2003 if (aTrack == TrackInfo::kAudioTrack) {
2004 decoder.mWorkingInfo = MakeUnique<AudioInfo>(*info->GetAsAudioInfo());
2005 } else if (aTrack == TrackInfo::kVideoTrack) {
2006 decoder.mWorkingInfo = MakeUnique<VideoInfo>(*info->GetAsVideoInfo());
2007 }
2008 }
2009
2010 decoder.mMeanRate.Reset();
2011
2012 if (sample->mKeyframe) {
2013 if (samples.Length()) {
2014 decoder.mQueuedSamples = std::move(samples);
2015 }
2016 } else {
2017 auto time = TimeInterval(sample->mTime, sample->GetEndTime());
2018 InternalSeekTarget seekTarget =
2019 decoder.mTimeThreshold.refOr(InternalSeekTarget(time, false));
2020 LOG("Stream change occurred on a non-keyframe. Seeking to:%" PRId64,
2021 sample->mTime.ToMicroseconds());
2022 InternalSeek(aTrack, seekTarget);
2023 return;
2024 }
2025 }
2026
2027 // Calculate the average frame rate. The first frame will be accounted
2028 // for twice.
2029 decoder.mMeanRate.Update(sample->mDuration);
2030
2031 if (!decoder.mDecoder) {
2032 mDecoderFactory->CreateDecoder(aTrack);
2033 return;
2034 }
2035
2036 LOGV("Input:%" PRId64 " (dts:%" PRId64 " kf:%d)",
2037 sample->mTime.ToMicroseconds(), sample->mTimecode.ToMicroseconds(),
2038 sample->mKeyframe);
2039 decoder.mNumSamplesInput++;
2040 decoder.mSizeOfQueue++;
2041 if (aTrack == TrackInfo::kVideoTrack) {
2042 aA.mStats.mParsedFrames++;
2043 }
2044
2045 DecodeDemuxedSamples(aTrack, sample);
2046
2047 decoder.mQueuedSamples.RemoveElementAt(0);
2048 }
2049
GetInternalSeekTargetEndTime() const2050 media::TimeUnit MediaFormatReader::GetInternalSeekTargetEndTime() const {
2051 MOZ_ASSERT(OnTaskQueue());
2052 return mVideo.mTimeThreshold ? mVideo.mTimeThreshold->EndTime()
2053 : TimeUnit::FromInfinity();
2054 }
2055
InternalSeek(TrackType aTrack,const InternalSeekTarget & aTarget)2056 void MediaFormatReader::InternalSeek(TrackType aTrack,
2057 const InternalSeekTarget& aTarget) {
2058 MOZ_ASSERT(OnTaskQueue());
2059 LOG("%s internal seek to %f", TrackTypeToStr(aTrack),
2060 aTarget.Time().ToSeconds());
2061
2062 auto& decoder = GetDecoderData(aTrack);
2063 decoder.Flush();
2064 decoder.ResetDemuxer();
2065 decoder.mTimeThreshold = Some(aTarget);
2066 DDLOG(DDLogCategory::Log, "seeking", DDNoValue{});
2067 RefPtr<MediaFormatReader> self = this;
2068 decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().Time())
2069 ->Then(
2070 OwnerThread(), __func__,
2071 [self, aTrack](TimeUnit aTime) {
2072 DDLOGEX(self.get(), DDLogCategory::Log, "seeked", DDNoValue{});
2073 auto& decoder = self->GetDecoderData(aTrack);
2074 decoder.mSeekRequest.Complete();
2075 MOZ_ASSERT(decoder.mTimeThreshold,
2076 "Seek promise must be disconnected when "
2077 "timethreshold is reset");
2078 decoder.mTimeThreshold.ref().mHasSeeked = true;
2079 self->SetVideoDecodeThreshold();
2080 self->ScheduleUpdate(aTrack);
2081 },
2082 [self, aTrack](const MediaResult& aError) {
2083 auto& decoder = self->GetDecoderData(aTrack);
2084 decoder.mSeekRequest.Complete();
2085 switch (aError.Code()) {
2086 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2087 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2088 aError);
2089 self->NotifyWaitingForData(aTrack);
2090 break;
2091 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2092 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2093 aError);
2094 decoder.mTimeThreshold.reset();
2095 self->NotifyEndOfStream(aTrack);
2096 break;
2097 case NS_ERROR_DOM_MEDIA_CANCELED:
2098 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_interrupted",
2099 aError);
2100 decoder.mTimeThreshold.reset();
2101 break;
2102 default:
2103 DDLOGEX(self.get(), DDLogCategory::Log, "seeking_error",
2104 aError);
2105 decoder.mTimeThreshold.reset();
2106 self->NotifyError(aTrack, aError);
2107 break;
2108 }
2109 })
2110 ->Track(decoder.mSeekRequest);
2111 }
2112
DrainDecoder(TrackType aTrack)2113 void MediaFormatReader::DrainDecoder(TrackType aTrack) {
2114 AUTO_PROFILER_LABEL("MediaFormatReader::DrainDecoder", MEDIA_PLAYBACK);
2115 MOZ_ASSERT(OnTaskQueue());
2116
2117 auto& decoder = GetDecoderData(aTrack);
2118 if (decoder.mDrainState == DrainState::Draining) {
2119 return;
2120 }
2121 if (!decoder.mDecoder ||
2122 (decoder.mDrainState != DrainState::PartialDrainPending &&
2123 decoder.mNumSamplesInput == decoder.mNumSamplesOutput)) {
2124 // No frames to drain.
2125 LOGV("Draining %s with nothing to drain", TrackTypeToStr(aTrack));
2126 decoder.mDrainState = DrainState::DrainAborted;
2127 ScheduleUpdate(aTrack);
2128 return;
2129 }
2130
2131 decoder.mDrainState = DrainState::Draining;
2132
2133 DDLOG(DDLogCategory::Log, "draining", DDNoValue{});
2134 RefPtr<MediaFormatReader> self = this;
2135 decoder.mDecoder->Drain()
2136 ->Then(
2137 mTaskQueue, __func__,
2138 [self, aTrack, &decoder](MediaDataDecoder::DecodedData&& aResults) {
2139 decoder.mDrainRequest.Complete();
2140 DDLOGEX(self.get(), DDLogCategory::Log, "drained", DDNoValue{});
2141 if (aResults.IsEmpty()) {
2142 decoder.mDrainState = DrainState::DrainCompleted;
2143 } else {
2144 self->NotifyNewOutput(aTrack, std::move(aResults));
2145 // Let's see if we have any more data available to drain.
2146 decoder.mDrainState = DrainState::PartialDrainPending;
2147 }
2148 self->ScheduleUpdate(aTrack);
2149 },
2150 [self, aTrack, &decoder](const MediaResult& aError) {
2151 decoder.mDrainRequest.Complete();
2152 DDLOGEX(self.get(), DDLogCategory::Log, "draining_error", aError);
2153 self->NotifyError(aTrack, aError);
2154 })
2155 ->Track(decoder.mDrainRequest);
2156 LOG("Requesting %s decoder to drain", TrackTypeToStr(aTrack));
2157 }
2158
Update(TrackType aTrack)2159 void MediaFormatReader::Update(TrackType aTrack) {
2160 AUTO_PROFILER_LABEL("MediaFormatReader::Update", MEDIA_PLAYBACK);
2161 MOZ_ASSERT(OnTaskQueue());
2162
2163 if (mShutdown) {
2164 return;
2165 }
2166
2167 LOGV("Processing update for %s", TrackTypeToStr(aTrack));
2168
2169 bool needOutput = false;
2170 auto& decoder = GetDecoderData(aTrack);
2171 decoder.mUpdateScheduled = false;
2172
2173 if (!mInitDone) {
2174 return;
2175 }
2176
2177 if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
2178 LOGV("Skipping in progress, nothing more to do");
2179 return;
2180 }
2181
2182 if (UpdateReceivedNewData(aTrack)) {
2183 LOGV("Nothing more to do");
2184 return;
2185 }
2186
2187 if (decoder.mSeekRequest.Exists()) {
2188 LOGV("Seeking hasn't completed, nothing more to do");
2189 return;
2190 }
2191
2192 MOZ_DIAGNOSTIC_ASSERT(
2193 !decoder.HasInternalSeekPending() ||
2194 (!decoder.mOutput.Length() && !decoder.mQueuedSamples.Length()),
2195 "No frames can be demuxed or decoded while an internal seek is pending");
2196
2197 // Record number of frames decoded and parsed. Automatically update the
2198 // stats counters using the AutoNotifyDecoded stack-based class.
2199 FrameStatistics::AutoNotifyDecoded a(mFrameStats);
2200
2201 // Drop any frames found prior our internal seek target.
2202 while (decoder.mTimeThreshold && decoder.mOutput.Length()) {
2203 RefPtr<MediaData>& output = decoder.mOutput[0];
2204 InternalSeekTarget target = decoder.mTimeThreshold.ref();
2205 auto time = output->mTime;
2206 if (time >= target.Time()) {
2207 // We have reached our internal seek target.
2208 decoder.mTimeThreshold.reset();
2209 // We might have dropped some keyframes.
2210 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2211 }
2212 if (time < target.Time() || (target.mDropTarget && target.Contains(time))) {
2213 LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
2214 TrackTypeToStr(aTrack), output->mTime.ToSeconds(),
2215 target.Time().ToSeconds(), output->mKeyframe);
2216 decoder.mOutput.RemoveElementAt(0);
2217 decoder.mSizeOfQueue -= 1;
2218 }
2219 }
2220
2221 while (decoder.mOutput.Length() &&
2222 decoder.mOutput[0]->mType == MediaData::Type::NULL_DATA) {
2223 LOGV("Dropping null data. Time: %" PRId64,
2224 decoder.mOutput[0]->mTime.ToMicroseconds());
2225 decoder.mOutput.RemoveElementAt(0);
2226 decoder.mSizeOfQueue -= 1;
2227 }
2228
2229 if (decoder.HasPromise()) {
2230 needOutput = true;
2231 if (decoder.mOutput.Length()) {
2232 RefPtr<MediaData> output = decoder.mOutput[0];
2233 decoder.mOutput.RemoveElementAt(0);
2234 decoder.mSizeOfQueue -= 1;
2235 decoder.mLastDecodedSampleTime =
2236 Some(TimeInterval(output->mTime, output->GetEndTime()));
2237 decoder.mNumSamplesOutputTotal++;
2238 ReturnOutput(output, aTrack);
2239 // We have a decoded sample ready to be returned.
2240 if (aTrack == TrackType::kVideoTrack) {
2241 uint64_t delta =
2242 decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
2243 a.mStats.mDecodedFrames = static_cast<uint32_t>(delta);
2244 mLastReportedNumDecodedFrames = decoder.mNumSamplesOutputTotal;
2245 if (output->mKeyframe) {
2246 if (mPreviousDecodedKeyframeTime_us <
2247 output->mTime.ToMicroseconds()) {
2248 // There is a previous keyframe -> Record inter-keyframe stats.
2249 uint64_t segment_us = output->mTime.ToMicroseconds() -
2250 mPreviousDecodedKeyframeTime_us;
2251 a.mStats.mInterKeyframeSum_us += segment_us;
2252 a.mStats.mInterKeyframeCount += 1;
2253 if (a.mStats.mInterKeyFrameMax_us < segment_us) {
2254 a.mStats.mInterKeyFrameMax_us = segment_us;
2255 }
2256 }
2257 mPreviousDecodedKeyframeTime_us = output->mTime.ToMicroseconds();
2258 }
2259 bool wasHardwareAccelerated = mVideo.mIsHardwareAccelerated;
2260 nsCString error;
2261 mVideo.mIsHardwareAccelerated =
2262 mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
2263 VideoData* videoData = output->As<VideoData>();
2264 if (!mVideo.mHasReportedVideoHardwareSupportTelemtry ||
2265 wasHardwareAccelerated != mVideo.mIsHardwareAccelerated) {
2266 mVideo.mHasReportedVideoHardwareSupportTelemtry = true;
2267 Telemetry::ScalarSet(
2268 Telemetry::ScalarID::MEDIA_VIDEO_HARDWARE_DECODING_SUPPORT,
2269 NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2270 !!mVideo.mIsHardwareAccelerated);
2271 static constexpr gfx::IntSize HD_VIDEO_SIZE{1280, 720};
2272 if (videoData->mDisplay.width >= HD_VIDEO_SIZE.Width() &&
2273 videoData->mDisplay.height >= HD_VIDEO_SIZE.Height()) {
2274 Telemetry::ScalarSet(
2275 Telemetry::ScalarID::MEDIA_VIDEO_HD_HARDWARE_DECODING_SUPPORT,
2276 NS_ConvertUTF8toUTF16(mVideo.GetCurrentInfo()->mMimeType),
2277 !!mVideo.mIsHardwareAccelerated);
2278 }
2279 }
2280 #ifdef XP_WIN
2281 // D3D11_YCBCR_IMAGE images are GPU based, we try to limit the amount
2282 // of GPU RAM used.
2283 mVideo.mIsHardwareAccelerated =
2284 mVideo.mIsHardwareAccelerated ||
2285 (videoData->mImage &&
2286 videoData->mImage->GetFormat() == ImageFormat::D3D11_YCBCR_IMAGE);
2287 #endif
2288 }
2289 } else if (decoder.HasFatalError()) {
2290 nsCString mimeType = decoder.GetCurrentInfo()->mMimeType;
2291 if (!mimeType.IsEmpty()) {
2292 Telemetry::ScalarAdd(
2293 Telemetry::ScalarID::MEDIA_DECODE_ERROR_PER_MIME_TYPE,
2294 NS_ConvertUTF8toUTF16(mimeType), 1 /* error count */);
2295 }
2296 LOG("Rejecting %s promise for %s : DECODE_ERROR", TrackTypeToStr(aTrack),
2297 mimeType.get());
2298 decoder.RejectPromise(decoder.mError.ref(), __func__);
2299 return;
2300 } else if (decoder.HasCompletedDrain()) {
2301 if (decoder.mDemuxEOS) {
2302 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2303 if (aTrack == TrackInfo::kVideoTrack) {
2304 // End of video, store the benchmark of the decoder.
2305 NotifyDecoderBenchmarkStore();
2306 }
2307 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2308 } else if (decoder.mWaitingForData) {
2309 if (decoder.mDrainState == DrainState::DrainCompleted &&
2310 decoder.mLastDecodedSampleTime && !decoder.mNextStreamSourceID) {
2311 // We have completed draining the decoder following WaitingForData.
2312 // Set up the internal seek machinery to be able to resume from the
2313 // last sample decoded.
2314 LOG("Seeking to last sample time: %" PRId64,
2315 decoder.mLastDecodedSampleTime.ref().mStart.ToMicroseconds());
2316 InternalSeek(aTrack, InternalSeekTarget(
2317 decoder.mLastDecodedSampleTime.ref(), true));
2318 }
2319 if (!decoder.mReceivedNewData) {
2320 LOG("Rejecting %s promise: WAITING_FOR_DATA", TrackTypeToStr(aTrack));
2321 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2322 }
2323 }
2324
2325 decoder.mDrainState = DrainState::None;
2326
2327 // Now that draining has completed, we check if we have received
2328 // new data again as the result may now be different from the earlier
2329 // run.
2330 if (UpdateReceivedNewData(aTrack) || decoder.mSeekRequest.Exists()) {
2331 LOGV("Nothing more to do");
2332 return;
2333 }
2334 } else if (decoder.mDemuxEOS && !decoder.HasPendingDrain() &&
2335 decoder.mQueuedSamples.IsEmpty()) {
2336 // It is possible to transition from WAITING_FOR_DATA directly to EOS
2337 // state during the internal seek; in which case no draining would occur.
2338 // There is no more samples left to be decoded and we are already in
2339 // EOS state. We can immediately reject the data promise.
2340 LOG("Rejecting %s promise: EOS", TrackTypeToStr(aTrack));
2341 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
2342 } else if (decoder.mWaitingForKey) {
2343 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for key",
2344 TrackTypeToStr(aTrack));
2345 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2346 } else if (IsDecoderWaitingForCDM(aTrack)) {
2347 // Rejecting the promise could lead to entering buffering state for MDSM,
2348 // once a qualified(with the same key system and sessions created by the
2349 // same InitData) new cdm proxy is set, decoding can be resumed.
2350 LOG("Rejecting %s promise: WAITING_FOR_DATA due to waiting for CDM",
2351 TrackTypeToStr(aTrack));
2352 decoder.RejectPromise(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
2353 }
2354 }
2355
2356 if (decoder.mDrainState == DrainState::DrainRequested ||
2357 decoder.mDrainState == DrainState::PartialDrainPending) {
2358 if (decoder.mOutput.IsEmpty()) {
2359 DrainDecoder(aTrack);
2360 }
2361 return;
2362 }
2363
2364 if (decoder.mError && !decoder.HasFatalError()) {
2365 MOZ_RELEASE_ASSERT(!decoder.HasInternalSeekPending(),
2366 "No error can occur while an internal seek is pending");
2367
2368 nsCString error;
2369 bool firstFrameDecodingFailedWithHardware =
2370 decoder.mFirstFrameTime &&
2371 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2372 decoder.mDecoder && decoder.mDecoder->IsHardwareAccelerated(error) &&
2373 !decoder.mHardwareDecodingDisabled;
2374 bool needsNewDecoder =
2375 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER ||
2376 decoder.mError.ref() == NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_ERR ||
2377 firstFrameDecodingFailedWithHardware;
2378 #ifdef XP_LINUX
2379 // We failed to decode on Linux with HW decoder,
2380 // give it another try without HW decoder.
2381 if (decoder.mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR &&
2382 decoder.mDecoder->IsHardwareAccelerated(error)) {
2383 LOG("Error: decode error, disable HW acceleration");
2384 needsNewDecoder = true;
2385 decoder.mHardwareDecodingDisabled = true;
2386 }
2387 // RDD process crashed on Linux, give it another try without HW decoder.
2388 if (decoder.mError.ref() == NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_ERR) {
2389 LOG("Error: remote decoder crashed, disable HW acceleration");
2390 decoder.mHardwareDecodingDisabled = true;
2391 }
2392 #endif
2393 // We don't want to expose NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_ERR
2394 // so switch to NS_ERROR_DOM_MEDIA_DECODE_ERR.
2395 if (decoder.mError.ref() == NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_ERR) {
2396 decoder.mError = Some(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
2397 RESULT_DETAIL("Unable to decode")));
2398 }
2399 if (!needsNewDecoder &&
2400 ++decoder.mNumOfConsecutiveError > decoder.mMaxConsecutiveError) {
2401 DDLOG(DDLogCategory::Log, "too_many_decode_errors", decoder.mError.ref());
2402 NotifyError(aTrack, decoder.mError.ref());
2403 return;
2404 }
2405 if (firstFrameDecodingFailedWithHardware) {
2406 decoder.mHardwareDecodingDisabled = true;
2407 }
2408 decoder.mError.reset();
2409
2410 LOG("%s decoded error count %d", TrackTypeToStr(aTrack),
2411 decoder.mNumOfConsecutiveError);
2412
2413 if (needsNewDecoder) {
2414 LOG("Error: Need new decoder");
2415 ShutdownDecoder(aTrack);
2416 }
2417 if (decoder.mFirstFrameTime) {
2418 TimeInterval seekInterval = TimeInterval(decoder.mFirstFrameTime.ref(),
2419 decoder.mFirstFrameTime.ref());
2420 InternalSeek(aTrack, InternalSeekTarget(seekInterval, false));
2421 return;
2422 }
2423
2424 TimeUnit nextKeyframe;
2425 if (aTrack == TrackType::kVideoTrack &&
2426 NS_SUCCEEDED(
2427 decoder.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe)) &&
2428 !nextKeyframe.IsInfinite()) {
2429 SkipVideoDemuxToNextKeyFrame(
2430 decoder.mLastDecodedSampleTime.refOr(TimeInterval()).Length());
2431 } else if (aTrack == TrackType::kAudioTrack) {
2432 decoder.Flush();
2433 } else {
2434 DDLOG(DDLogCategory::Log, "no_keyframe", NS_ERROR_DOM_MEDIA_FATAL_ERR);
2435 // We can't recover from this error.
2436 NotifyError(aTrack, NS_ERROR_DOM_MEDIA_FATAL_ERR);
2437 }
2438 return;
2439 }
2440
2441 bool needInput = NeedInput(decoder);
2442
2443 LOGV("Update(%s) ni=%d no=%d in:%" PRIu64 " out:%" PRIu64
2444 " qs=%u decoding:%d flushing:%d desc:%s pending:%u waiting:%d eos:%d "
2445 "ds:%d sid:%u waitcdm:%d",
2446 TrackTypeToStr(aTrack), needInput, needOutput, decoder.mNumSamplesInput,
2447 decoder.mNumSamplesOutput, uint32_t(size_t(decoder.mSizeOfQueue)),
2448 decoder.mDecodeRequest.Exists(), decoder.mFlushing,
2449 decoder.mDescription.get(), uint32_t(decoder.mOutput.Length()),
2450 decoder.mWaitingForData, decoder.mDemuxEOS, int32_t(decoder.mDrainState),
2451 decoder.mLastStreamSourceID, IsDecoderWaitingForCDM(aTrack));
2452
2453 if (IsWaitingOnCDMResource() || !ResolveSetCDMPromiseIfDone(aTrack)) {
2454 // If the content is encrypted, MFR won't start to create decoder until
2455 // CDMProxy is set.
2456 return;
2457 }
2458
2459 if ((decoder.IsWaitingForData() &&
2460 (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) ||
2461 (decoder.IsWaitingForKey())) {
2462 // Nothing more we can do at present.
2463 LOGV("Still waiting for data or key. data(%d)/key(%d)",
2464 decoder.mWaitingForData, decoder.mWaitingForKey);
2465 return;
2466 }
2467
2468 if (decoder.CancelWaitingForKey()) {
2469 LOGV("No longer waiting for key. Resolving waiting promise");
2470 return;
2471 }
2472
2473 if (!needInput) {
2474 LOGV("No need for additional input (pending:%u)",
2475 uint32_t(decoder.mOutput.Length()));
2476 return;
2477 }
2478
2479 // Demux samples if we don't have some.
2480 RequestDemuxSamples(aTrack);
2481
2482 HandleDemuxedSamples(aTrack, a);
2483 }
2484
ReturnOutput(MediaData * aData,TrackType aTrack)2485 void MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack) {
2486 AUTO_PROFILER_LABEL("MediaFormatReader::ReturnOutput", MEDIA_PLAYBACK);
2487 MOZ_ASSERT(GetDecoderData(aTrack).HasPromise());
2488 MOZ_DIAGNOSTIC_ASSERT(aData->mType != MediaData::Type::NULL_DATA);
2489 LOG("Resolved data promise for %s [%" PRId64 ", %" PRId64 "]",
2490 TrackTypeToStr(aTrack), aData->mTime.ToMicroseconds(),
2491 aData->GetEndTime().ToMicroseconds());
2492
2493 if (aTrack == TrackInfo::kAudioTrack) {
2494 AudioData* audioData = aData->As<AudioData>();
2495
2496 if (audioData->mChannels != mInfo.mAudio.mChannels ||
2497 audioData->mRate != mInfo.mAudio.mRate) {
2498 LOG("change of audio format (rate:%d->%d). "
2499 "This is an unsupported configuration",
2500 mInfo.mAudio.mRate, audioData->mRate);
2501 mInfo.mAudio.mRate = audioData->mRate;
2502 mInfo.mAudio.mChannels = audioData->mChannels;
2503 MutexAutoLock lock(mAudio.mMutex);
2504 mAudio.mWorkingInfo->GetAsAudioInfo()->mRate = audioData->mRate;
2505 mAudio.mWorkingInfo->GetAsAudioInfo()->mChannels = audioData->mChannels;
2506 }
2507 mAudio.ResolvePromise(audioData, __func__);
2508 } else if (aTrack == TrackInfo::kVideoTrack) {
2509 VideoData* videoData = aData->As<VideoData>();
2510
2511 if (videoData->mDisplay != mInfo.mVideo.mDisplay) {
2512 LOG("change of video display size (%dx%d->%dx%d)",
2513 mInfo.mVideo.mDisplay.width, mInfo.mVideo.mDisplay.height,
2514 videoData->mDisplay.width, videoData->mDisplay.height);
2515 mInfo.mVideo.mDisplay = videoData->mDisplay;
2516 MutexAutoLock lock(mVideo.mMutex);
2517 mVideo.mWorkingInfo->GetAsVideoInfo()->mDisplay = videoData->mDisplay;
2518 }
2519
2520 TimeUnit nextKeyframe;
2521 if (!mVideo.HasInternalSeekPending() &&
2522 NS_SUCCEEDED(
2523 mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe))) {
2524 videoData->SetNextKeyFrameTime(nextKeyframe);
2525 }
2526
2527 mVideo.ResolvePromise(videoData, __func__);
2528 }
2529 }
2530
SizeOfVideoQueueInFrames()2531 size_t MediaFormatReader::SizeOfVideoQueueInFrames() {
2532 return SizeOfQueue(TrackInfo::kVideoTrack);
2533 }
2534
SizeOfAudioQueueInFrames()2535 size_t MediaFormatReader::SizeOfAudioQueueInFrames() {
2536 return SizeOfQueue(TrackInfo::kAudioTrack);
2537 }
2538
SizeOfQueue(TrackType aTrack)2539 size_t MediaFormatReader::SizeOfQueue(TrackType aTrack) {
2540 auto& decoder = GetDecoderData(aTrack);
2541 return decoder.mSizeOfQueue;
2542 }
2543
WaitForData(MediaData::Type aType)2544 RefPtr<MediaFormatReader::WaitForDataPromise> MediaFormatReader::WaitForData(
2545 MediaData::Type aType) {
2546 MOZ_ASSERT(OnTaskQueue());
2547 TrackType trackType = aType == MediaData::Type::VIDEO_DATA
2548 ? TrackType::kVideoTrack
2549 : TrackType::kAudioTrack;
2550 auto& decoder = GetDecoderData(trackType);
2551 if (!decoder.IsWaitingForData() && !decoder.IsWaitingForKey()) {
2552 // We aren't waiting for anything.
2553 return WaitForDataPromise::CreateAndResolve(decoder.mType, __func__);
2554 }
2555 RefPtr<WaitForDataPromise> p = decoder.mWaitingPromise.Ensure(__func__);
2556 ScheduleUpdate(trackType);
2557 return p;
2558 }
2559
ResetDecode(TrackSet aTracks)2560 nsresult MediaFormatReader::ResetDecode(TrackSet aTracks) {
2561 AUTO_PROFILER_LABEL("MediaFormatReader::ResetDecode", MEDIA_PLAYBACK);
2562 MOZ_ASSERT(OnTaskQueue());
2563 LOGV("");
2564
2565 mSeekPromise.RejectIfExists(NS_OK, __func__);
2566 mSkipRequest.DisconnectIfExists();
2567
2568 // Do the same for any data wait promises.
2569 if (aTracks.contains(TrackInfo::kAudioTrack)) {
2570 mAudio.mWaitingPromise.RejectIfExists(
2571 WaitForDataRejectValue(MediaData::Type::AUDIO_DATA,
2572 WaitForDataRejectValue::CANCELED),
2573 __func__);
2574 }
2575
2576 if (aTracks.contains(TrackInfo::kVideoTrack)) {
2577 mVideo.mWaitingPromise.RejectIfExists(
2578 WaitForDataRejectValue(MediaData::Type::VIDEO_DATA,
2579 WaitForDataRejectValue::CANCELED),
2580 __func__);
2581 }
2582
2583 // Reset miscellaneous seeking state.
2584 mPendingSeekTime.reset();
2585
2586 if (HasVideo() && aTracks.contains(TrackInfo::kVideoTrack)) {
2587 mVideo.ResetDemuxer();
2588 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2589 Reset(TrackInfo::kVideoTrack);
2590 if (mVideo.HasPromise()) {
2591 mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2592 }
2593 }
2594
2595 if (HasAudio() && aTracks.contains(TrackInfo::kAudioTrack)) {
2596 mAudio.ResetDemuxer();
2597 mVideo.mFirstFrameTime = Some(media::TimeUnit::Zero());
2598 Reset(TrackInfo::kAudioTrack);
2599 if (mAudio.HasPromise()) {
2600 mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
2601 }
2602 }
2603
2604 return NS_OK;
2605 }
2606
Reset(TrackType aTrack)2607 void MediaFormatReader::Reset(TrackType aTrack) {
2608 MOZ_ASSERT(OnTaskQueue());
2609 LOG("Reset(%s) BEGIN", TrackTypeToStr(aTrack));
2610
2611 auto& decoder = GetDecoderData(aTrack);
2612
2613 decoder.ResetState();
2614 decoder.Flush();
2615
2616 LOG("Reset(%s) END", TrackTypeToStr(aTrack));
2617 }
2618
DropDecodedSamples(TrackType aTrack)2619 void MediaFormatReader::DropDecodedSamples(TrackType aTrack) {
2620 MOZ_ASSERT(OnTaskQueue());
2621 auto& decoder = GetDecoderData(aTrack);
2622 size_t lengthDecodedQueue = decoder.mOutput.Length();
2623 if (lengthDecodedQueue && decoder.mTimeThreshold.isSome()) {
2624 auto time = decoder.mOutput.LastElement()->mTime;
2625 if (time >= decoder.mTimeThreshold.ref().Time()) {
2626 // We would have reached our internal seek target.
2627 decoder.mTimeThreshold.reset();
2628 }
2629 }
2630 decoder.mOutput.Clear();
2631 decoder.mSizeOfQueue -= lengthDecodedQueue;
2632 if (aTrack == TrackInfo::kVideoTrack && mFrameStats) {
2633 mFrameStats->Accumulate({0, 0, 0, lengthDecodedQueue, 0, 0});
2634 }
2635 }
2636
SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold)2637 void MediaFormatReader::SkipVideoDemuxToNextKeyFrame(TimeUnit aTimeThreshold) {
2638 AUTO_PROFILER_LABEL("MediaFormatReader::SkipVideoDemuxToNextKeyFrame",
2639 MEDIA_PLAYBACK);
2640 MOZ_ASSERT(OnTaskQueue());
2641 LOG("Skipping up to %" PRId64, aTimeThreshold.ToMicroseconds());
2642
2643 // We've reached SkipVideoDemuxToNextKeyFrame when our decoding is late.
2644 // As such we can drop all already decoded samples and discard all pending
2645 // samples.
2646 DropDecodedSamples(TrackInfo::kVideoTrack);
2647
2648 mVideo.mTrackDemuxer->SkipToNextRandomAccessPoint(aTimeThreshold)
2649 ->Then(OwnerThread(), __func__, this,
2650 &MediaFormatReader::OnVideoSkipCompleted,
2651 &MediaFormatReader::OnVideoSkipFailed)
2652 ->Track(mSkipRequest);
2653 }
2654
VideoSkipReset(uint32_t aSkipped)2655 void MediaFormatReader::VideoSkipReset(uint32_t aSkipped) {
2656 PROFILER_MARKER_UNTYPED("SkippedVideoDecode", MEDIA_PLAYBACK);
2657 MOZ_ASSERT(OnTaskQueue());
2658
2659 // Some frames may have been output by the decoder since we initiated the
2660 // videoskip process and we know they would be late.
2661 DropDecodedSamples(TrackInfo::kVideoTrack);
2662 // Report the pending frames as dropped.
2663 if (mFrameStats) {
2664 uint32_t droppedDecoderCount = SizeOfVideoQueueInFrames();
2665 mFrameStats->Accumulate({0, 0, 0, droppedDecoderCount, 0, 0});
2666 }
2667
2668 // Cancel any pending demux request and pending demuxed samples.
2669 mVideo.mDemuxRequest.DisconnectIfExists();
2670 Reset(TrackType::kVideoTrack);
2671
2672 if (mFrameStats) {
2673 mFrameStats->Accumulate({aSkipped, 0, 0, aSkipped, 0, 0});
2674 }
2675
2676 mVideo.mNumSamplesSkippedTotal += aSkipped;
2677 }
2678
OnVideoSkipCompleted(uint32_t aSkipped)2679 void MediaFormatReader::OnVideoSkipCompleted(uint32_t aSkipped) {
2680 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipCompleted",
2681 MEDIA_PLAYBACK);
2682 MOZ_ASSERT(OnTaskQueue());
2683 LOG("Skipping succeeded, skipped %u frames", aSkipped);
2684 mSkipRequest.Complete();
2685
2686 DDLOG(DDLogCategory::Log, "video_skipped", DDNoValue());
2687
2688 VideoSkipReset(aSkipped);
2689
2690 ScheduleUpdate(TrackInfo::kVideoTrack);
2691 }
2692
OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure)2693 void MediaFormatReader::OnVideoSkipFailed(
2694 MediaTrackDemuxer::SkipFailureHolder aFailure) {
2695 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSkipFailed", MEDIA_PLAYBACK);
2696 MOZ_ASSERT(OnTaskQueue());
2697 LOG("Skipping failed, skipped %u frames", aFailure.mSkipped);
2698 mSkipRequest.Complete();
2699
2700 switch (aFailure.mFailure.Code()) {
2701 case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
2702 case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
2703 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2704 aFailure.mFailure);
2705 // Some frames may have been output by the decoder since we initiated the
2706 // videoskip process and we know they would be late.
2707 DropDecodedSamples(TrackInfo::kVideoTrack);
2708 // We can't complete the skip operation, will just service a video frame
2709 // normally.
2710 ScheduleUpdate(TrackInfo::kVideoTrack);
2711 break;
2712 case NS_ERROR_DOM_MEDIA_CANCELED:
2713 DDLOG(DDLogCategory::Log, "video_skipping_interruption",
2714 aFailure.mFailure);
2715 if (mVideo.HasPromise()) {
2716 mVideo.RejectPromise(aFailure.mFailure, __func__);
2717 }
2718 break;
2719 default:
2720 DDLOG(DDLogCategory::Log, "video_skipping_error", aFailure.mFailure);
2721 NotifyError(TrackType::kVideoTrack, aFailure.mFailure);
2722 break;
2723 }
2724 }
2725
Seek(const SeekTarget & aTarget)2726 RefPtr<MediaFormatReader::SeekPromise> MediaFormatReader::Seek(
2727 const SeekTarget& aTarget) {
2728 AUTO_PROFILER_LABEL("MediaFormatReader::Seek", MEDIA_PLAYBACK);
2729 MOZ_ASSERT(OnTaskQueue());
2730
2731 LOG("aTarget=(%" PRId64 ")", aTarget.GetTime().ToMicroseconds());
2732
2733 MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty());
2734 MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise());
2735 MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() || !mAudio.HasPromise());
2736 MOZ_DIAGNOSTIC_ASSERT(mPendingSeekTime.isNothing());
2737 MOZ_DIAGNOSTIC_ASSERT(mVideo.mTimeThreshold.isNothing());
2738 MOZ_DIAGNOSTIC_ASSERT(aTarget.IsVideoOnly() ||
2739 mAudio.mTimeThreshold.isNothing());
2740
2741 if (!mInfo.mMediaSeekable && !mInfo.mMediaSeekableOnlyInBufferedRanges) {
2742 LOG("Seek() END (Unseekable)");
2743 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2744 }
2745
2746 if (mShutdown) {
2747 return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
2748 }
2749
2750 SetSeekTarget(aTarget);
2751
2752 RefPtr<SeekPromise> p = mSeekPromise.Ensure(__func__);
2753
2754 ScheduleSeek();
2755
2756 return p;
2757 }
2758
SetSeekTarget(const SeekTarget & aTarget)2759 void MediaFormatReader::SetSeekTarget(const SeekTarget& aTarget) {
2760 MOZ_ASSERT(OnTaskQueue());
2761
2762 mOriginalSeekTarget = aTarget;
2763 mFallbackSeekTime = mPendingSeekTime = Some(aTarget.GetTime());
2764 }
2765
ScheduleSeek()2766 void MediaFormatReader::ScheduleSeek() {
2767 if (mSeekScheduled) {
2768 return;
2769 }
2770 mSeekScheduled = true;
2771 nsresult rv = OwnerThread()->Dispatch(NewRunnableMethod(
2772 "MediaFormatReader::AttemptSeek", this, &MediaFormatReader::AttemptSeek));
2773 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
2774 Unused << rv;
2775 }
2776
AttemptSeek()2777 void MediaFormatReader::AttemptSeek() {
2778 AUTO_PROFILER_LABEL("MediaFormatReader::AttemptSeek", MEDIA_PLAYBACK);
2779 MOZ_ASSERT(OnTaskQueue());
2780
2781 mSeekScheduled = false;
2782
2783 if (mPendingSeekTime.isNothing()) {
2784 return;
2785 }
2786
2787 if (HasVideo()) {
2788 mVideo.ResetDemuxer();
2789 mVideo.ResetState();
2790 }
2791
2792 // Don't reset the audio demuxer not state when seeking video only
2793 // as it will cause the audio to seek back to the beginning
2794 // resulting in out-of-sync audio from video.
2795 if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
2796 mAudio.ResetDemuxer();
2797 mAudio.ResetState();
2798 }
2799
2800 if (HasVideo()) {
2801 DoVideoSeek();
2802 } else if (HasAudio()) {
2803 DoAudioSeek();
2804 } else {
2805 MOZ_CRASH();
2806 }
2807 }
2808
OnSeekFailed(TrackType aTrack,const MediaResult & aError)2809 void MediaFormatReader::OnSeekFailed(TrackType aTrack,
2810 const MediaResult& aError) {
2811 AUTO_PROFILER_LABEL("MediaFormatReader::OnSeekFailed", MEDIA_PLAYBACK);
2812 MOZ_ASSERT(OnTaskQueue());
2813 LOGV("%s failure:%s", TrackTypeToStr(aTrack), aError.ErrorName().get());
2814 if (aTrack == TrackType::kVideoTrack) {
2815 mVideo.mSeekRequest.Complete();
2816 } else {
2817 mAudio.mSeekRequest.Complete();
2818 }
2819
2820 if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
2821 if (HasVideo() && aTrack == TrackType::kAudioTrack &&
2822 mFallbackSeekTime.isSome() &&
2823 mPendingSeekTime.ref() != mFallbackSeekTime.ref()) {
2824 // We have failed to seek audio where video seeked to earlier.
2825 // Attempt to seek instead to the closest point that we know we have in
2826 // order to limit A/V sync discrepency.
2827
2828 // Ensure we have the most up to date buffered ranges.
2829 UpdateReceivedNewData(TrackType::kAudioTrack);
2830 Maybe<TimeUnit> nextSeekTime;
2831 // Find closest buffered time found after video seeked time.
2832 for (const auto& timeRange : mAudio.mTimeRanges) {
2833 if (timeRange.mStart >= mPendingSeekTime.ref()) {
2834 nextSeekTime.emplace(timeRange.mStart);
2835 break;
2836 }
2837 }
2838 if (nextSeekTime.isNothing() ||
2839 nextSeekTime.ref() > mFallbackSeekTime.ref()) {
2840 nextSeekTime = Some(mFallbackSeekTime.ref());
2841 LOG("Unable to seek audio to video seek time. A/V sync may be broken");
2842 } else {
2843 mFallbackSeekTime.reset();
2844 }
2845 mPendingSeekTime = nextSeekTime;
2846 DoAudioSeek();
2847 return;
2848 }
2849 NotifyWaitingForData(aTrack);
2850 }
2851 MOZ_ASSERT(!mVideo.mSeekRequest.Exists() && !mAudio.mSeekRequest.Exists());
2852 mPendingSeekTime.reset();
2853
2854 auto type = aTrack == TrackType::kAudioTrack ? MediaData::Type::AUDIO_DATA
2855 : MediaData::Type::VIDEO_DATA;
2856 mSeekPromise.Reject(SeekRejectValue(type, aError), __func__);
2857 }
2858
DoVideoSeek()2859 void MediaFormatReader::DoVideoSeek() {
2860 AUTO_PROFILER_LABEL("MediaFormatReader::DoVideoSeek", MEDIA_PLAYBACK);
2861 MOZ_ASSERT(mPendingSeekTime.isSome());
2862 LOGV("Seeking video to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
2863 auto seekTime = mPendingSeekTime.ref();
2864 mVideo.mTrackDemuxer->Seek(seekTime)
2865 ->Then(OwnerThread(), __func__, this,
2866 &MediaFormatReader::OnVideoSeekCompleted,
2867 &MediaFormatReader::OnVideoSeekFailed)
2868 ->Track(mVideo.mSeekRequest);
2869 }
2870
OnVideoSeekCompleted(TimeUnit aTime)2871 void MediaFormatReader::OnVideoSeekCompleted(TimeUnit aTime) {
2872 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekCompleted",
2873 MEDIA_PLAYBACK);
2874 MOZ_ASSERT(OnTaskQueue());
2875 LOGV("Video seeked to %" PRId64, aTime.ToMicroseconds());
2876 mVideo.mSeekRequest.Complete();
2877
2878 mVideo.mFirstFrameTime = Some(aTime);
2879 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2880
2881 SetVideoDecodeThreshold();
2882
2883 if (HasAudio() && !mOriginalSeekTarget.IsVideoOnly()) {
2884 MOZ_ASSERT(mPendingSeekTime.isSome());
2885 if (mOriginalSeekTarget.IsFast()) {
2886 // We are performing a fast seek. We need to seek audio to where the
2887 // video seeked to, to ensure proper A/V sync once playback resume.
2888 mPendingSeekTime = Some(aTime);
2889 }
2890 DoAudioSeek();
2891 } else {
2892 mPendingSeekTime.reset();
2893 mSeekPromise.Resolve(aTime, __func__);
2894 }
2895 }
2896
OnVideoSeekFailed(const MediaResult & aError)2897 void MediaFormatReader::OnVideoSeekFailed(const MediaResult& aError) {
2898 AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoSeekFailed", MEDIA_PLAYBACK);
2899 mPreviousDecodedKeyframeTime_us = sNoPreviousDecodedKeyframe;
2900 OnSeekFailed(TrackType::kVideoTrack, aError);
2901 }
2902
SetVideoDecodeThreshold()2903 void MediaFormatReader::SetVideoDecodeThreshold() {
2904 MOZ_ASSERT(OnTaskQueue());
2905
2906 if (!HasVideo() || !mVideo.mDecoder) {
2907 return;
2908 }
2909
2910 if (!mVideo.mTimeThreshold && !IsSeeking()) {
2911 return;
2912 }
2913
2914 TimeUnit threshold;
2915 if (mVideo.mTimeThreshold) {
2916 // For internalSeek.
2917 threshold = mVideo.mTimeThreshold.ref().Time();
2918 } else if (IsSeeking()) {
2919 // If IsSeeking() is true, then video seek must have completed already.
2920 TimeUnit keyframe;
2921 if (NS_FAILED(mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&keyframe))) {
2922 return;
2923 }
2924
2925 // If the key frame is invalid/infinite, it means the target position is
2926 // closing to end of stream. We don't want to skip any frame at this point.
2927 threshold = keyframe.IsValid() && !keyframe.IsInfinite()
2928 ? mOriginalSeekTarget.GetTime()
2929 : TimeUnit::Invalid();
2930 } else {
2931 return;
2932 }
2933
2934 if (threshold.IsValid()) {
2935 LOG("Set seek threshold to %" PRId64, threshold.ToMicroseconds());
2936 } else {
2937 LOG("Resetting seek threshold");
2938 }
2939 mVideo.mDecoder->SetSeekThreshold(threshold);
2940 }
2941
DoAudioSeek()2942 void MediaFormatReader::DoAudioSeek() {
2943 AUTO_PROFILER_LABEL("MediaFormatReader::DoAudioSeek", MEDIA_PLAYBACK);
2944 MOZ_ASSERT(mPendingSeekTime.isSome());
2945 LOGV("Seeking audio to %" PRId64, mPendingSeekTime.ref().ToMicroseconds());
2946 auto seekTime = mPendingSeekTime.ref();
2947 mAudio.mTrackDemuxer->Seek(seekTime)
2948 ->Then(OwnerThread(), __func__, this,
2949 &MediaFormatReader::OnAudioSeekCompleted,
2950 &MediaFormatReader::OnAudioSeekFailed)
2951 ->Track(mAudio.mSeekRequest);
2952 }
2953
OnAudioSeekCompleted(TimeUnit aTime)2954 void MediaFormatReader::OnAudioSeekCompleted(TimeUnit aTime) {
2955 MOZ_ASSERT(OnTaskQueue());
2956 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekCompleted",
2957 MEDIA_PLAYBACK);
2958 LOGV("Audio seeked to %" PRId64, aTime.ToMicroseconds());
2959 mAudio.mSeekRequest.Complete();
2960 mAudio.mFirstFrameTime = Some(aTime);
2961 mPendingSeekTime.reset();
2962 mSeekPromise.Resolve(aTime, __func__);
2963 }
2964
OnAudioSeekFailed(const MediaResult & aError)2965 void MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError) {
2966 AUTO_PROFILER_LABEL("MediaFormatReader::OnAudioSeekFailed", MEDIA_PLAYBACK);
2967 OnSeekFailed(TrackType::kAudioTrack, aError);
2968 }
2969
ReleaseResources()2970 void MediaFormatReader::ReleaseResources() {
2971 LOGV("");
2972 if (mShutdown) {
2973 return;
2974 }
2975 ShutdownDecoder(TrackInfo::kAudioTrack);
2976 ShutdownDecoder(TrackInfo::kVideoTrack);
2977 }
2978
VideoIsHardwareAccelerated() const2979 bool MediaFormatReader::VideoIsHardwareAccelerated() const {
2980 return mVideo.mIsHardwareAccelerated;
2981 }
2982
NotifyTrackDemuxers()2983 void MediaFormatReader::NotifyTrackDemuxers() {
2984 MOZ_ASSERT(OnTaskQueue());
2985
2986 LOGV("");
2987
2988 if (!mInitDone) {
2989 return;
2990 }
2991
2992 if (HasVideo()) {
2993 mVideo.mReceivedNewData = true;
2994 ScheduleUpdate(TrackType::kVideoTrack);
2995 }
2996 if (HasAudio()) {
2997 mAudio.mReceivedNewData = true;
2998 ScheduleUpdate(TrackType::kAudioTrack);
2999 }
3000 }
3001
NotifyDataArrived()3002 void MediaFormatReader::NotifyDataArrived() {
3003 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived", MEDIA_PLAYBACK);
3004 MOZ_ASSERT(OnTaskQueue());
3005
3006 if (mShutdown || !mDemuxer || !mDemuxerInitDone) {
3007 return;
3008 }
3009
3010 if (mNotifyDataArrivedPromise.Exists()) {
3011 // Already one in progress. Set the dirty flag so we can process it later.
3012 mPendingNotifyDataArrived = true;
3013 return;
3014 }
3015
3016 RefPtr<MediaFormatReader> self = this;
3017 mDemuxer->NotifyDataArrived()
3018 ->Then(
3019 OwnerThread(), __func__,
3020 [self]() {
3021 AUTO_PROFILER_LABEL("MediaFormatReader::NotifyDataArrived:Resolved",
3022 MEDIA_PLAYBACK);
3023 self->mNotifyDataArrivedPromise.Complete();
3024 self->UpdateBuffered();
3025 self->NotifyTrackDemuxers();
3026 if (self->mPendingNotifyDataArrived) {
3027 self->mPendingNotifyDataArrived = false;
3028 self->NotifyDataArrived();
3029 }
3030 },
3031 [self]() { self->mNotifyDataArrivedPromise.Complete(); })
3032 ->Track(mNotifyDataArrivedPromise);
3033 }
3034
UpdateBuffered()3035 void MediaFormatReader::UpdateBuffered() {
3036 AUTO_PROFILER_LABEL("MediaFormatReader::UpdateBuffered", MEDIA_PLAYBACK);
3037 MOZ_ASSERT(OnTaskQueue());
3038
3039 if (mShutdown) {
3040 return;
3041 }
3042
3043 if (!mInitDone || !mHasStartTime) {
3044 mBuffered = TimeIntervals();
3045 return;
3046 }
3047
3048 if (HasVideo()) {
3049 mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered();
3050 bool hasLastEnd;
3051 auto lastEnd = mVideo.mTimeRanges.GetEnd(&hasLastEnd);
3052 if (hasLastEnd) {
3053 if (mVideo.mLastTimeRangesEnd &&
3054 mVideo.mLastTimeRangesEnd.ref() < lastEnd) {
3055 // New data was added after our previous end, we can clear the EOS flag.
3056 mVideo.mDemuxEOS = false;
3057 ScheduleUpdate(TrackInfo::kVideoTrack);
3058 }
3059 mVideo.mLastTimeRangesEnd = Some(lastEnd);
3060 }
3061 }
3062 if (HasAudio()) {
3063 mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered();
3064 bool hasLastEnd;
3065 auto lastEnd = mAudio.mTimeRanges.GetEnd(&hasLastEnd);
3066 if (hasLastEnd) {
3067 if (mAudio.mLastTimeRangesEnd &&
3068 mAudio.mLastTimeRangesEnd.ref() < lastEnd) {
3069 // New data was added after our previous end, we can clear the EOS flag.
3070 mAudio.mDemuxEOS = false;
3071 ScheduleUpdate(TrackInfo::kAudioTrack);
3072 }
3073 mAudio.mLastTimeRangesEnd = Some(lastEnd);
3074 }
3075 }
3076
3077 media::TimeIntervals intervals;
3078 if (HasAudio() && HasVideo()) {
3079 intervals = media::Intersection(mVideo.mTimeRanges, mAudio.mTimeRanges);
3080 } else if (HasAudio()) {
3081 intervals = mAudio.mTimeRanges;
3082 } else if (HasVideo()) {
3083 intervals = mVideo.mTimeRanges;
3084 }
3085
3086 if (intervals.IsEmpty() || intervals.GetStart() == TimeUnit::Zero()) {
3087 // IntervalSet already starts at 0 or is empty, nothing to shift.
3088 mBuffered = intervals;
3089 } else {
3090 mBuffered = intervals.Shift(TimeUnit::Zero() - mInfo.mStartTime);
3091 }
3092 }
3093
GetImageContainer()3094 layers::ImageContainer* MediaFormatReader::GetImageContainer() {
3095 return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
3096 : nullptr;
3097 }
3098
GetDebugInfo(dom::MediaFormatReaderDebugInfo & aInfo)3099 void MediaFormatReader::GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo) {
3100 nsCString result;
3101 nsAutoCString audioDecoderName("unavailable");
3102 nsAutoCString videoDecoderName = audioDecoderName;
3103 nsAutoCString audioType("none");
3104 nsAutoCString videoType("none");
3105
3106 AudioInfo audioInfo;
3107 {
3108 MutexAutoLock lock(mAudio.mMutex);
3109 if (HasAudio()) {
3110 audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
3111 audioDecoderName = mAudio.mDecoder ? mAudio.mDecoder->GetDescriptionName()
3112 : mAudio.mDescription;
3113 audioType = audioInfo.mMimeType;
3114 }
3115 }
3116
3117 VideoInfo videoInfo;
3118 {
3119 MutexAutoLock lock(mVideo.mMutex);
3120 if (HasVideo()) {
3121 videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
3122 videoDecoderName = mVideo.mDecoder ? mVideo.mDecoder->GetDescriptionName()
3123 : mVideo.mDescription;
3124 videoType = videoInfo.mMimeType;
3125 }
3126 }
3127
3128 CopyUTF8toUTF16(audioDecoderName, aInfo.mAudioDecoderName);
3129 CopyUTF8toUTF16(audioType, aInfo.mAudioType);
3130 aInfo.mAudioChannels = audioInfo.mChannels;
3131 aInfo.mAudioRate = audioInfo.mRate / 1000.0f;
3132 aInfo.mAudioFramesDecoded = mAudio.mNumSamplesOutputTotal;
3133
3134 if (HasAudio()) {
3135 aInfo.mAudioState.mNeedInput = NeedInput(mAudio);
3136 aInfo.mAudioState.mHasPromise = mAudio.HasPromise();
3137 aInfo.mAudioState.mWaitingPromise = !mAudio.mWaitingPromise.IsEmpty();
3138 aInfo.mAudioState.mHasDemuxRequest = mAudio.mDemuxRequest.Exists();
3139 aInfo.mAudioState.mDemuxQueueSize =
3140 uint32_t(mAudio.mQueuedSamples.Length());
3141 aInfo.mAudioState.mHasDecoder = mAudio.mDecodeRequest.Exists();
3142 aInfo.mAudioState.mTimeTreshold =
3143 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().Time().ToSeconds()
3144 : -1.0;
3145 aInfo.mAudioState.mTimeTresholdHasSeeked =
3146 mAudio.mTimeThreshold ? mAudio.mTimeThreshold.ref().mHasSeeked : false;
3147 aInfo.mAudioState.mNumSamplesInput = mAudio.mNumSamplesInput;
3148 aInfo.mAudioState.mNumSamplesOutput = mAudio.mNumSamplesOutput;
3149 aInfo.mAudioState.mQueueSize = size_t(mAudio.mSizeOfQueue);
3150 aInfo.mAudioState.mPending = mAudio.mOutput.Length();
3151 aInfo.mAudioState.mWaitingForData = mAudio.mWaitingForData;
3152 aInfo.mAudioState.mDemuxEOS = mAudio.mDemuxEOS;
3153 aInfo.mAudioState.mDrainState = int32_t(mAudio.mDrainState);
3154 aInfo.mAudioState.mWaitingForKey = mAudio.mWaitingForKey;
3155 aInfo.mAudioState.mLastStreamSourceID = mAudio.mLastStreamSourceID;
3156 }
3157
3158 CopyUTF8toUTF16(videoDecoderName, aInfo.mVideoDecoderName);
3159 CopyUTF8toUTF16(videoType, aInfo.mVideoType);
3160 aInfo.mVideoWidth =
3161 videoInfo.mDisplay.width < 0 ? 0 : videoInfo.mDisplay.width;
3162 aInfo.mVideoHeight =
3163 videoInfo.mDisplay.height < 0 ? 0 : videoInfo.mDisplay.height;
3164 aInfo.mVideoRate = mVideo.mMeanRate.Mean();
3165 aInfo.mVideoHardwareAccelerated = VideoIsHardwareAccelerated();
3166 aInfo.mVideoNumSamplesOutputTotal = mVideo.mNumSamplesOutputTotal;
3167 aInfo.mVideoNumSamplesSkippedTotal = mVideo.mNumSamplesSkippedTotal;
3168
3169 if (HasVideo()) {
3170 aInfo.mVideoState.mNeedInput = NeedInput(mVideo);
3171 aInfo.mVideoState.mHasPromise = mVideo.HasPromise();
3172 aInfo.mVideoState.mWaitingPromise = !mVideo.mWaitingPromise.IsEmpty();
3173 aInfo.mVideoState.mHasDemuxRequest = mVideo.mDemuxRequest.Exists();
3174 aInfo.mVideoState.mDemuxQueueSize =
3175 uint32_t(mVideo.mQueuedSamples.Length());
3176 aInfo.mVideoState.mHasDecoder = mVideo.mDecodeRequest.Exists();
3177 aInfo.mVideoState.mTimeTreshold =
3178 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().Time().ToSeconds()
3179 : -1.0;
3180 aInfo.mVideoState.mTimeTresholdHasSeeked =
3181 mVideo.mTimeThreshold ? mVideo.mTimeThreshold.ref().mHasSeeked : false;
3182 aInfo.mVideoState.mNumSamplesInput = mVideo.mNumSamplesInput;
3183 aInfo.mVideoState.mNumSamplesOutput = mVideo.mNumSamplesOutput;
3184 aInfo.mVideoState.mQueueSize = size_t(mVideo.mSizeOfQueue);
3185 aInfo.mVideoState.mPending = mVideo.mOutput.Length();
3186 aInfo.mVideoState.mWaitingForData = mVideo.mWaitingForData;
3187 aInfo.mVideoState.mDemuxEOS = mVideo.mDemuxEOS;
3188 aInfo.mVideoState.mDrainState = int32_t(mVideo.mDrainState);
3189 aInfo.mVideoState.mWaitingForKey = mVideo.mWaitingForKey;
3190 aInfo.mVideoState.mLastStreamSourceID = mVideo.mLastStreamSourceID;
3191 }
3192
3193 // Looking at dropped frames
3194 FrameStatisticsData stats = mFrameStats->GetFrameStatisticsData();
3195 aInfo.mFrameStats.mDroppedDecodedFrames = stats.mDroppedDecodedFrames;
3196 aInfo.mFrameStats.mDroppedSinkFrames = stats.mDroppedSinkFrames;
3197 aInfo.mFrameStats.mDroppedCompositorFrames = stats.mDroppedCompositorFrames;
3198 }
3199
SetVideoNullDecode(bool aIsNullDecode)3200 void MediaFormatReader::SetVideoNullDecode(bool aIsNullDecode) {
3201 MOZ_ASSERT(OnTaskQueue());
3202 return SetNullDecode(TrackType::kVideoTrack, aIsNullDecode);
3203 }
3204
UpdateCompositor(already_AddRefed<layers::KnowsCompositor> aCompositor)3205 void MediaFormatReader::UpdateCompositor(
3206 already_AddRefed<layers::KnowsCompositor> aCompositor) {
3207 MOZ_ASSERT(OnTaskQueue());
3208 mKnowsCompositor = aCompositor;
3209 }
3210
SetNullDecode(TrackType aTrack,bool aIsNullDecode)3211 void MediaFormatReader::SetNullDecode(TrackType aTrack, bool aIsNullDecode) {
3212 MOZ_ASSERT(OnTaskQueue());
3213
3214 auto& decoder = GetDecoderData(aTrack);
3215 if (decoder.mIsNullDecode == aIsNullDecode) {
3216 return;
3217 }
3218
3219 LOG("%s, decoder.mIsNullDecode = %d => aIsNullDecode = %d",
3220 TrackTypeToStr(aTrack), decoder.mIsNullDecode, aIsNullDecode);
3221
3222 decoder.mIsNullDecode = aIsNullDecode;
3223 ShutdownDecoder(aTrack);
3224 }
3225
OnFirstDemuxCompleted(TrackInfo::TrackType aType,RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples)3226 void MediaFormatReader::OnFirstDemuxCompleted(
3227 TrackInfo::TrackType aType,
3228 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
3229 AUTO_PROFILER_LABEL("MediaFormatReader::OnFirstDemuxCompleted",
3230 MEDIA_PLAYBACK);
3231 MOZ_ASSERT(OnTaskQueue());
3232
3233 if (mShutdown) {
3234 return;
3235 }
3236
3237 auto& decoder = GetDecoderData(aType);
3238 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3239 decoder.mFirstDemuxedSampleTime.emplace(aSamples->GetSamples()[0]->mTime);
3240 MaybeResolveMetadataPromise();
3241 }
3242
OnFirstDemuxFailed(TrackInfo::TrackType aType,const MediaResult & aError)3243 void MediaFormatReader::OnFirstDemuxFailed(TrackInfo::TrackType aType,
3244 const MediaResult& aError) {
3245 MOZ_ASSERT(OnTaskQueue());
3246
3247 if (mShutdown) {
3248 return;
3249 }
3250
3251 auto& decoder = GetDecoderData(aType);
3252 MOZ_ASSERT(decoder.mFirstDemuxedSampleTime.isNothing());
3253 decoder.mFirstDemuxedSampleTime.emplace(TimeUnit::FromInfinity());
3254 MaybeResolveMetadataPromise();
3255 }
3256
3257 } // namespace mozilla
3258
3259 #undef NS_DispatchToMainThread
3260 #undef LOGV
3261 #undef LOG
3262