1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaDecoder.h"
8
9 #include "AudioDeviceInfo.h"
10 #include "DOMMediaStream.h"
11 #include "DecoderBenchmark.h"
12 #include "ImageContainer.h"
13 #include "Layers.h"
14 #include "MediaDecoderStateMachine.h"
15 #include "MediaFormatReader.h"
16 #include "MediaResource.h"
17 #include "MediaShutdownManager.h"
18 #include "MediaTrackGraph.h"
19 #include "TelemetryProbesReporter.h"
20 #include "VideoFrameContainer.h"
21 #include "VideoUtils.h"
22 #include "mozilla/AbstractThread.h"
23 #include "mozilla/dom/DOMTypes.h"
24 #include "mozilla/FloatingPoint.h"
25 #include "mozilla/MathAlgorithms.h"
26 #include "mozilla/Preferences.h"
27 #include "mozilla/StaticPrefs_media.h"
28 #include "mozilla/StaticPtr.h"
29 #include "mozilla/Telemetry.h"
30 #include "mozilla/Unused.h"
31 #include "nsComponentManagerUtils.h"
32 #include "nsContentUtils.h"
33 #include "nsError.h"
34 #include "nsIMemoryReporter.h"
35 #include "nsPrintfCString.h"
36 #include "nsServiceManagerUtils.h"
37 #include "nsTArray.h"
38 #include <algorithm>
39 #include <limits>
40
41 using namespace mozilla::dom;
42 using namespace mozilla::layers;
43 using namespace mozilla::media;
44
45 namespace mozilla {
46
47 // avoid redefined macro in unified build
48 #undef LOG
49 #undef DUMP
50
51 LazyLogModule gMediaDecoderLog("MediaDecoder");
52 #define LOG(x, ...) \
53 DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, x, ##__VA_ARGS__)
54
55 #define DUMP(x, ...) printf_stderr(x "\n", ##__VA_ARGS__)
56
57 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
58
ToPlayStateStr(MediaDecoder::PlayState aState)59 static const char* ToPlayStateStr(MediaDecoder::PlayState aState) {
60 switch (aState) {
61 case MediaDecoder::PLAY_STATE_LOADING:
62 return "LOADING";
63 case MediaDecoder::PLAY_STATE_PAUSED:
64 return "PAUSED";
65 case MediaDecoder::PLAY_STATE_PLAYING:
66 return "PLAYING";
67 case MediaDecoder::PLAY_STATE_ENDED:
68 return "ENDED";
69 case MediaDecoder::PLAY_STATE_SHUTDOWN:
70 return "SHUTDOWN";
71 default:
72 MOZ_ASSERT_UNREACHABLE("Invalid playState.");
73 }
74 return "UNKNOWN";
75 }
76
77 class MediaMemoryTracker : public nsIMemoryReporter {
78 virtual ~MediaMemoryTracker();
79
80 NS_DECL_THREADSAFE_ISUPPORTS
81 NS_DECL_NSIMEMORYREPORTER
82
83 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
84
85 MediaMemoryTracker();
86 void InitMemoryReporter();
87
88 static StaticRefPtr<MediaMemoryTracker> sUniqueInstance;
89
UniqueInstance()90 static MediaMemoryTracker* UniqueInstance() {
91 if (!sUniqueInstance) {
92 sUniqueInstance = new MediaMemoryTracker();
93 sUniqueInstance->InitMemoryReporter();
94 }
95 return sUniqueInstance;
96 }
97
98 typedef nsTArray<MediaDecoder*> DecodersArray;
Decoders()99 static DecodersArray& Decoders() { return UniqueInstance()->mDecoders; }
100
101 DecodersArray mDecoders;
102
103 public:
AddMediaDecoder(MediaDecoder * aDecoder)104 static void AddMediaDecoder(MediaDecoder* aDecoder) {
105 Decoders().AppendElement(aDecoder);
106 }
107
RemoveMediaDecoder(MediaDecoder * aDecoder)108 static void RemoveMediaDecoder(MediaDecoder* aDecoder) {
109 DecodersArray& decoders = Decoders();
110 decoders.RemoveElement(aDecoder);
111 if (decoders.IsEmpty()) {
112 sUniqueInstance = nullptr;
113 }
114 }
115
GetSizes()116 static RefPtr<MediaMemoryPromise> GetSizes() {
117 MOZ_ASSERT(NS_IsMainThread());
118 DecodersArray& decoders = Decoders();
119
120 // if we don't have any decoder, we can bail
121 if (decoders.IsEmpty()) {
122 // and release the instance that was created by calling Decoders()
123 sUniqueInstance = nullptr;
124 return MediaMemoryPromise::CreateAndResolve(MediaMemoryInfo(), __func__);
125 }
126
127 RefPtr<MediaDecoder::ResourceSizes> resourceSizes =
128 new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf);
129
130 size_t videoSize = 0;
131 size_t audioSize = 0;
132
133 for (auto&& decoder : decoders) {
134 videoSize += decoder->SizeOfVideoQueue();
135 audioSize += decoder->SizeOfAudioQueue();
136 decoder->AddSizeOfResources(resourceSizes);
137 }
138
139 return resourceSizes->Promise()->Then(
140 AbstractThread::MainThread(), __func__,
141 [videoSize, audioSize](size_t resourceSize) {
142 return MediaMemoryPromise::CreateAndResolve(
143 MediaMemoryInfo(videoSize, audioSize, resourceSize), __func__);
144 },
145 [](size_t) {
146 return MediaMemoryPromise::CreateAndReject(NS_ERROR_FAILURE,
147 __func__);
148 });
149 }
150 };
151
152 StaticRefPtr<MediaMemoryTracker> MediaMemoryTracker::sUniqueInstance;
153
GetMediaMemorySizes()154 RefPtr<MediaMemoryPromise> GetMediaMemorySizes() {
155 return MediaMemoryTracker::GetSizes();
156 }
157
158 LazyLogModule gMediaTimerLog("MediaTimer");
159
160 constexpr TimeUnit MediaDecoder::DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED;
161
InitStatics()162 void MediaDecoder::InitStatics() {
163 MOZ_ASSERT(NS_IsMainThread());
164 // Eagerly init gMediaDecoderLog to work around bug 1415441.
165 MOZ_LOG(gMediaDecoderLog, LogLevel::Info, ("MediaDecoder::InitStatics"));
166 }
167
NS_IMPL_ISUPPORTS(MediaMemoryTracker,nsIMemoryReporter)168 NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter)
169
170 void MediaDecoder::NotifyOwnerActivityChanged(bool aIsOwnerInvisible,
171 bool aIsOwnerConnected) {
172 MOZ_ASSERT(NS_IsMainThread());
173 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
174 SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected);
175
176 NotifyCompositor();
177 }
178
Pause()179 void MediaDecoder::Pause() {
180 MOZ_ASSERT(NS_IsMainThread());
181 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
182 LOG("Pause");
183 if (mPlayState == PLAY_STATE_LOADING || IsEnded()) {
184 mNextState = PLAY_STATE_PAUSED;
185 return;
186 }
187 ChangeState(PLAY_STATE_PAUSED);
188 }
189
SetVolume(double aVolume)190 void MediaDecoder::SetVolume(double aVolume) {
191 MOZ_ASSERT(NS_IsMainThread());
192 mVolume = aVolume;
193 }
194
SetSink(AudioDeviceInfo * aSinkDevice)195 RefPtr<GenericPromise> MediaDecoder::SetSink(AudioDeviceInfo* aSinkDevice) {
196 MOZ_ASSERT(NS_IsMainThread());
197 mSinkDevice = aSinkDevice;
198 return GetStateMachine()->InvokeSetSink(aSinkDevice);
199 }
200
SetOutputCaptureState(OutputCaptureState aState,SharedDummyTrack * aDummyTrack)201 void MediaDecoder::SetOutputCaptureState(OutputCaptureState aState,
202 SharedDummyTrack* aDummyTrack) {
203 MOZ_ASSERT(NS_IsMainThread());
204 MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
205 MOZ_ASSERT_IF(aState == OutputCaptureState::Capture, aDummyTrack);
206 mOutputCaptureState = aState;
207 if (mOutputDummyTrack.Ref().get() != aDummyTrack) {
208 mOutputDummyTrack = nsMainThreadPtrHandle<SharedDummyTrack>(
209 MakeAndAddRef<nsMainThreadPtrHolder<SharedDummyTrack>>(
210 "MediaDecoder::mOutputDummyTrack", aDummyTrack));
211 }
212 }
213
AddOutputTrack(RefPtr<ProcessedMediaTrack> aTrack)214 void MediaDecoder::AddOutputTrack(RefPtr<ProcessedMediaTrack> aTrack) {
215 MOZ_ASSERT(NS_IsMainThread());
216 MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
217 CopyableTArray<RefPtr<ProcessedMediaTrack>> tracks = mOutputTracks;
218 tracks.AppendElement(std::move(aTrack));
219 mOutputTracks = tracks;
220 }
221
RemoveOutputTrack(const RefPtr<ProcessedMediaTrack> & aTrack)222 void MediaDecoder::RemoveOutputTrack(
223 const RefPtr<ProcessedMediaTrack>& aTrack) {
224 MOZ_ASSERT(NS_IsMainThread());
225 MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
226 CopyableTArray<RefPtr<ProcessedMediaTrack>> tracks = mOutputTracks;
227 if (tracks.RemoveElement(aTrack)) {
228 mOutputTracks = tracks;
229 }
230 }
231
SetOutputTracksPrincipal(const RefPtr<nsIPrincipal> & aPrincipal)232 void MediaDecoder::SetOutputTracksPrincipal(
233 const RefPtr<nsIPrincipal>& aPrincipal) {
234 MOZ_ASSERT(NS_IsMainThread());
235 MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
236 mOutputPrincipal = MakePrincipalHandle(aPrincipal);
237 }
238
GetDuration()239 double MediaDecoder::GetDuration() {
240 MOZ_ASSERT(NS_IsMainThread());
241 return mDuration;
242 }
243
IsInfinite() const244 bool MediaDecoder::IsInfinite() const {
245 MOZ_ASSERT(NS_IsMainThread());
246 return mozilla::IsInfinite<double>(mDuration);
247 }
248
249 #define INIT_MIRROR(name, val) \
250 name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Mirror)")
251 #define INIT_CANONICAL(name, val) \
252 name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Canonical)")
253
MediaDecoder(MediaDecoderInit & aInit)254 MediaDecoder::MediaDecoder(MediaDecoderInit& aInit)
255 : mWatchManager(this, aInit.mOwner->AbstractMainThread()),
256 mLogicalPosition(0.0),
257 mDuration(std::numeric_limits<double>::quiet_NaN()),
258 mOwner(aInit.mOwner),
259 mAbstractMainThread(aInit.mOwner->AbstractMainThread()),
260 mFrameStats(new FrameStatistics()),
261 mDecoderBenchmark(new DecoderBenchmark()),
262 mVideoFrameContainer(aInit.mOwner->GetVideoFrameContainer()),
263 mMinimizePreroll(aInit.mMinimizePreroll),
264 mFiredMetadataLoaded(false),
265 mIsOwnerInvisible(false),
266 mIsOwnerConnected(false),
267 mForcedHidden(false),
268 mHasSuspendTaint(aInit.mHasSuspendTaint),
269 mPlaybackRate(aInit.mPlaybackRate),
270 mLogicallySeeking(false, "MediaDecoder::mLogicallySeeking"),
271 INIT_MIRROR(mBuffered, TimeIntervals()),
272 INIT_MIRROR(mCurrentPosition, TimeUnit::Zero()),
273 INIT_MIRROR(mStateMachineDuration, NullableTimeUnit()),
274 INIT_MIRROR(mIsAudioDataAudible, false),
275 INIT_CANONICAL(mVolume, aInit.mVolume),
276 INIT_CANONICAL(mPreservesPitch, aInit.mPreservesPitch),
277 INIT_CANONICAL(mLooping, aInit.mLooping),
278 INIT_CANONICAL(mStreamName, aInit.mStreamName),
279 INIT_CANONICAL(mSinkDevice, nullptr),
280 INIT_CANONICAL(mSecondaryVideoContainer, nullptr),
281 INIT_CANONICAL(mOutputCaptureState, OutputCaptureState::None),
282 INIT_CANONICAL(mOutputDummyTrack, nullptr),
283 INIT_CANONICAL(mOutputTracks, nsTArray<RefPtr<ProcessedMediaTrack>>()),
284 INIT_CANONICAL(mOutputPrincipal, PRINCIPAL_HANDLE_NONE),
285 INIT_CANONICAL(mPlayState, PLAY_STATE_LOADING),
286 mSameOriginMedia(false),
287 mVideoDecodingOberver(
288 new BackgroundVideoDecodingPermissionObserver(this)),
289 mIsBackgroundVideoDecodingAllowed(false),
290 mTelemetryReported(false),
291 mContainerType(aInit.mContainerType),
292 mTelemetryProbesReporter(
293 new TelemetryProbesReporter(aInit.mReporterOwner)) {
294 MOZ_ASSERT(NS_IsMainThread());
295 MOZ_ASSERT(mAbstractMainThread);
296 MediaMemoryTracker::AddMediaDecoder(this);
297
298 //
299 // Initialize watchers.
300 //
301
302 // mDuration
303 mWatchManager.Watch(mStateMachineDuration, &MediaDecoder::DurationChanged);
304
305 // readyState
306 mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateReadyState);
307 // ReadyState computation depends on MediaDecoder::CanPlayThrough, which
308 // depends on the download rate.
309 mWatchManager.Watch(mBuffered, &MediaDecoder::UpdateReadyState);
310
311 // mLogicalPosition
312 mWatchManager.Watch(mCurrentPosition, &MediaDecoder::UpdateLogicalPosition);
313 mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateLogicalPosition);
314 mWatchManager.Watch(mLogicallySeeking, &MediaDecoder::UpdateLogicalPosition);
315
316 mWatchManager.Watch(mIsAudioDataAudible,
317 &MediaDecoder::NotifyAudibleStateChanged);
318
319 mVideoDecodingOberver->RegisterEvent();
320 }
321
322 #undef INIT_MIRROR
323 #undef INIT_CANONICAL
324
Shutdown()325 void MediaDecoder::Shutdown() {
326 MOZ_ASSERT(NS_IsMainThread());
327 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
328
329 // Unwatch all watch targets to prevent further notifications.
330 mWatchManager.Shutdown();
331
332 DiscardOngoingSeekIfExists();
333
334 // This changes the decoder state to SHUTDOWN and does other things
335 // necessary to unblock the state machine thread if it's blocked, so
336 // the asynchronous shutdown in nsDestroyStateMachine won't deadlock.
337 if (mDecoderStateMachine) {
338 mTimedMetadataListener.Disconnect();
339 mMetadataLoadedListener.Disconnect();
340 mFirstFrameLoadedListener.Disconnect();
341 mOnPlaybackEvent.Disconnect();
342 mOnPlaybackErrorEvent.Disconnect();
343 mOnDecoderDoctorEvent.Disconnect();
344 mOnMediaNotSeekable.Disconnect();
345 mOnEncrypted.Disconnect();
346 mOnWaitingForKey.Disconnect();
347 mOnDecodeWarning.Disconnect();
348 mOnNextFrameStatus.Disconnect();
349 mOnSecondaryVideoContainerInstalled.Disconnect();
350 mOnStoreDecoderBenchmark.Disconnect();
351
352 mDecoderStateMachine->BeginShutdown()->Then(
353 mAbstractMainThread, __func__, this, &MediaDecoder::FinishShutdown,
354 &MediaDecoder::FinishShutdown);
355 } else {
356 // Ensure we always unregister asynchronously in order not to disrupt
357 // the hashtable iterating in MediaShutdownManager::Shutdown().
358 RefPtr<MediaDecoder> self = this;
359 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
360 "MediaDecoder::Shutdown", [self]() { self->ShutdownInternal(); });
361 mAbstractMainThread->Dispatch(r.forget());
362 }
363
364 ChangeState(PLAY_STATE_SHUTDOWN);
365 mVideoDecodingOberver->UnregisterEvent();
366 mVideoDecodingOberver = nullptr;
367 mOwner = nullptr;
368 }
369
NotifyXPCOMShutdown()370 void MediaDecoder::NotifyXPCOMShutdown() {
371 MOZ_ASSERT(NS_IsMainThread());
372 // NotifyXPCOMShutdown will clear its reference to mDecoder. So we must ensure
373 // that this MediaDecoder stays alive until completion.
374 RefPtr<MediaDecoder> kungFuDeathGrip = this;
375 if (auto owner = GetOwner()) {
376 owner->NotifyXPCOMShutdown();
377 } else if (!IsShutdown()) {
378 Shutdown();
379 }
380 MOZ_DIAGNOSTIC_ASSERT(IsShutdown());
381 }
382
~MediaDecoder()383 MediaDecoder::~MediaDecoder() {
384 MOZ_ASSERT(NS_IsMainThread());
385 MOZ_DIAGNOSTIC_ASSERT(IsShutdown());
386 MediaMemoryTracker::RemoveMediaDecoder(this);
387 }
388
OnPlaybackEvent(MediaPlaybackEvent && aEvent)389 void MediaDecoder::OnPlaybackEvent(MediaPlaybackEvent&& aEvent) {
390 switch (aEvent.mType) {
391 case MediaPlaybackEvent::PlaybackEnded:
392 PlaybackEnded();
393 break;
394 case MediaPlaybackEvent::SeekStarted:
395 SeekingStarted();
396 break;
397 case MediaPlaybackEvent::Invalidate:
398 Invalidate();
399 break;
400 case MediaPlaybackEvent::EnterVideoSuspend:
401 GetOwner()->DispatchAsyncEvent(u"mozentervideosuspend"_ns);
402 mTelemetryProbesReporter->OnDecodeSuspended();
403 mIsVideoDecodingSuspended = true;
404 break;
405 case MediaPlaybackEvent::ExitVideoSuspend:
406 GetOwner()->DispatchAsyncEvent(u"mozexitvideosuspend"_ns);
407 mTelemetryProbesReporter->OnDecodeResumed();
408 mIsVideoDecodingSuspended = false;
409 break;
410 case MediaPlaybackEvent::StartVideoSuspendTimer:
411 GetOwner()->DispatchAsyncEvent(u"mozstartvideosuspendtimer"_ns);
412 break;
413 case MediaPlaybackEvent::CancelVideoSuspendTimer:
414 GetOwner()->DispatchAsyncEvent(u"mozcancelvideosuspendtimer"_ns);
415 break;
416 case MediaPlaybackEvent::VideoOnlySeekBegin:
417 GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekbegin"_ns);
418 break;
419 case MediaPlaybackEvent::VideoOnlySeekCompleted:
420 GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekcompleted"_ns);
421 break;
422 default:
423 break;
424 }
425 }
426
IsVideoDecodingSuspended() const427 bool MediaDecoder::IsVideoDecodingSuspended() const {
428 return mIsVideoDecodingSuspended;
429 }
430
OnPlaybackErrorEvent(const MediaResult & aError)431 void MediaDecoder::OnPlaybackErrorEvent(const MediaResult& aError) {
432 DecodeError(aError);
433 }
434
OnDecoderDoctorEvent(DecoderDoctorEvent aEvent)435 void MediaDecoder::OnDecoderDoctorEvent(DecoderDoctorEvent aEvent) {
436 MOZ_ASSERT(NS_IsMainThread());
437 // OnDecoderDoctorEvent is disconnected at shutdown time.
438 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
439 Document* doc = GetOwner()->GetDocument();
440 if (!doc) {
441 return;
442 }
443 DecoderDoctorDiagnostics diags;
444 diags.StoreEvent(doc, aEvent, __func__);
445 }
446
NextFrameStatusToStr(MediaDecoderOwner::NextFrameStatus aStatus)447 static const char* NextFrameStatusToStr(
448 MediaDecoderOwner::NextFrameStatus aStatus) {
449 switch (aStatus) {
450 case MediaDecoderOwner::NEXT_FRAME_AVAILABLE:
451 return "NEXT_FRAME_AVAILABLE";
452 case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE:
453 return "NEXT_FRAME_UNAVAILABLE";
454 case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING:
455 return "NEXT_FRAME_UNAVAILABLE_BUFFERING";
456 case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING:
457 return "NEXT_FRAME_UNAVAILABLE_SEEKING";
458 case MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED:
459 return "NEXT_FRAME_UNINITIALIZED";
460 }
461 return "UNKNOWN";
462 }
463
OnNextFrameStatus(MediaDecoderOwner::NextFrameStatus aStatus)464 void MediaDecoder::OnNextFrameStatus(
465 MediaDecoderOwner::NextFrameStatus aStatus) {
466 MOZ_ASSERT(NS_IsMainThread());
467 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
468 if (mNextFrameStatus != aStatus) {
469 LOG("Changed mNextFrameStatus to %s", NextFrameStatusToStr(aStatus));
470 mNextFrameStatus = aStatus;
471 UpdateReadyState();
472 }
473 }
474
OnSecondaryVideoContainerInstalled(const RefPtr<VideoFrameContainer> & aSecondaryVideoContainer)475 void MediaDecoder::OnSecondaryVideoContainerInstalled(
476 const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer) {
477 MOZ_ASSERT(NS_IsMainThread());
478 GetOwner()->OnSecondaryVideoContainerInstalled(aSecondaryVideoContainer);
479 }
480
OnStoreDecoderBenchmark(const VideoInfo & aInfo)481 void MediaDecoder::OnStoreDecoderBenchmark(const VideoInfo& aInfo) {
482 MOZ_ASSERT(NS_IsMainThread());
483
484 int32_t videoFrameRate = aInfo.GetFrameRate().ref();
485
486 if (mFrameStats && videoFrameRate) {
487 DecoderBenchmarkInfo benchmarkInfo{
488 aInfo.mMimeType,
489 aInfo.mDisplay.width,
490 aInfo.mDisplay.height,
491 videoFrameRate,
492 BitDepthForColorDepth(aInfo.mColorDepth),
493 };
494
495 LOG("Store benchmark: Video width=%d, height=%d, frameRate=%d, content "
496 "type = %s\n",
497 benchmarkInfo.mWidth, benchmarkInfo.mHeight, benchmarkInfo.mFrameRate,
498 benchmarkInfo.mContentType.BeginReading());
499
500 mDecoderBenchmark->Store(benchmarkInfo, mFrameStats);
501 }
502 }
503
ShutdownInternal()504 void MediaDecoder::ShutdownInternal() {
505 MOZ_ASSERT(NS_IsMainThread());
506 mVideoFrameContainer = nullptr;
507 mSecondaryVideoContainer = nullptr;
508 MediaShutdownManager::Instance().Unregister(this);
509 }
510
FinishShutdown()511 void MediaDecoder::FinishShutdown() {
512 MOZ_ASSERT(NS_IsMainThread());
513 SetStateMachine(nullptr);
514 ShutdownInternal();
515 }
516
InitializeStateMachine()517 nsresult MediaDecoder::InitializeStateMachine() {
518 MOZ_ASSERT(NS_IsMainThread());
519 NS_ASSERTION(mDecoderStateMachine, "Cannot initialize null state machine!");
520
521 nsresult rv = mDecoderStateMachine->Init(this);
522 NS_ENSURE_SUCCESS(rv, rv);
523
524 // If some parameters got set before the state machine got created,
525 // set them now
526 SetStateMachineParameters();
527
528 return NS_OK;
529 }
530
SetStateMachineParameters()531 void MediaDecoder::SetStateMachineParameters() {
532 MOZ_ASSERT(NS_IsMainThread());
533 if (mPlaybackRate != 1 && mPlaybackRate != 0) {
534 mDecoderStateMachine->DispatchSetPlaybackRate(mPlaybackRate);
535 }
536 mTimedMetadataListener = mDecoderStateMachine->TimedMetadataEvent().Connect(
537 mAbstractMainThread, this, &MediaDecoder::OnMetadataUpdate);
538 mMetadataLoadedListener = mDecoderStateMachine->MetadataLoadedEvent().Connect(
539 mAbstractMainThread, this, &MediaDecoder::MetadataLoaded);
540 mFirstFrameLoadedListener =
541 mDecoderStateMachine->FirstFrameLoadedEvent().Connect(
542 mAbstractMainThread, this, &MediaDecoder::FirstFrameLoaded);
543
544 mOnPlaybackEvent = mDecoderStateMachine->OnPlaybackEvent().Connect(
545 mAbstractMainThread, this, &MediaDecoder::OnPlaybackEvent);
546 mOnPlaybackErrorEvent = mDecoderStateMachine->OnPlaybackErrorEvent().Connect(
547 mAbstractMainThread, this, &MediaDecoder::OnPlaybackErrorEvent);
548 mOnDecoderDoctorEvent = mDecoderStateMachine->OnDecoderDoctorEvent().Connect(
549 mAbstractMainThread, this, &MediaDecoder::OnDecoderDoctorEvent);
550 mOnMediaNotSeekable = mDecoderStateMachine->OnMediaNotSeekable().Connect(
551 mAbstractMainThread, this, &MediaDecoder::OnMediaNotSeekable);
552 mOnNextFrameStatus = mDecoderStateMachine->OnNextFrameStatus().Connect(
553 mAbstractMainThread, this, &MediaDecoder::OnNextFrameStatus);
554 mOnSecondaryVideoContainerInstalled =
555 mDecoderStateMachine->OnSecondaryVideoContainerInstalled().Connect(
556 mAbstractMainThread, this,
557 &MediaDecoder::OnSecondaryVideoContainerInstalled);
558 mOnStoreDecoderBenchmark = mReader->OnStoreDecoderBenchmark().Connect(
559 mAbstractMainThread, this, &MediaDecoder::OnStoreDecoderBenchmark);
560
561 mOnEncrypted = mReader->OnEncrypted().Connect(
562 mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DispatchEncrypted);
563 mOnWaitingForKey = mReader->OnWaitingForKey().Connect(
564 mAbstractMainThread, GetOwner(), &MediaDecoderOwner::NotifyWaitingForKey);
565 mOnDecodeWarning = mReader->OnDecodeWarning().Connect(
566 mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DecodeWarning);
567 }
568
Play()569 void MediaDecoder::Play() {
570 MOZ_ASSERT(NS_IsMainThread());
571
572 NS_ASSERTION(mDecoderStateMachine != nullptr, "Should have state machine.");
573 LOG("Play");
574 if (mPlaybackRate == 0) {
575 return;
576 }
577
578 if (IsEnded()) {
579 Seek(0, SeekTarget::PrevSyncPoint);
580 return;
581 } else if (mPlayState == PLAY_STATE_LOADING) {
582 mNextState = PLAY_STATE_PLAYING;
583 return;
584 }
585
586 ChangeState(PLAY_STATE_PLAYING);
587 }
588
Seek(double aTime,SeekTarget::Type aSeekType)589 void MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) {
590 MOZ_ASSERT(NS_IsMainThread());
591 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
592
593 MOZ_ASSERT(aTime >= 0.0, "Cannot seek to a negative value.");
594
595 LOG("Seek");
596 auto time = TimeUnit::FromSeconds(aTime);
597
598 mLogicalPosition = aTime;
599 mLogicallySeeking = true;
600 SeekTarget target = SeekTarget(time, aSeekType);
601 CallSeek(target);
602
603 if (mPlayState == PLAY_STATE_ENDED) {
604 ChangeState(GetOwner()->GetPaused() ? PLAY_STATE_PAUSED
605 : PLAY_STATE_PLAYING);
606 }
607 }
608
SetDelaySeekMode(bool aShouldDelaySeek)609 void MediaDecoder::SetDelaySeekMode(bool aShouldDelaySeek) {
610 MOZ_ASSERT(NS_IsMainThread());
611 LOG("SetDelaySeekMode, shouldDelaySeek=%d", aShouldDelaySeek);
612 if (mShouldDelaySeek == aShouldDelaySeek) {
613 return;
614 }
615 mShouldDelaySeek = aShouldDelaySeek;
616 if (!mShouldDelaySeek && mDelayedSeekTarget) {
617 Seek(mDelayedSeekTarget->GetTime().ToSeconds(),
618 mDelayedSeekTarget->GetType());
619 mDelayedSeekTarget.reset();
620 }
621 }
622
DiscardOngoingSeekIfExists()623 void MediaDecoder::DiscardOngoingSeekIfExists() {
624 MOZ_ASSERT(NS_IsMainThread());
625 mSeekRequest.DisconnectIfExists();
626 }
627
CallSeek(const SeekTarget & aTarget)628 void MediaDecoder::CallSeek(const SeekTarget& aTarget) {
629 MOZ_ASSERT(NS_IsMainThread());
630 if (mShouldDelaySeek) {
631 LOG("Delay seek to %f and store it to delayed seek target",
632 mDelayedSeekTarget->GetTime().ToSeconds());
633 mDelayedSeekTarget = Some(aTarget);
634 return;
635 }
636 DiscardOngoingSeekIfExists();
637 mDecoderStateMachine->InvokeSeek(aTarget)
638 ->Then(mAbstractMainThread, __func__, this, &MediaDecoder::OnSeekResolved,
639 &MediaDecoder::OnSeekRejected)
640 ->Track(mSeekRequest);
641 }
642
GetCurrentTime()643 double MediaDecoder::GetCurrentTime() {
644 MOZ_ASSERT(NS_IsMainThread());
645 return mLogicalPosition;
646 }
647
OnMetadataUpdate(TimedMetadata && aMetadata)648 void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) {
649 MOZ_ASSERT(NS_IsMainThread());
650 MetadataLoaded(MakeUnique<MediaInfo>(*aMetadata.mInfo),
651 UniquePtr<MetadataTags>(std::move(aMetadata.mTags)),
652 MediaDecoderEventVisibility::Observable);
653 FirstFrameLoaded(std::move(aMetadata.mInfo),
654 MediaDecoderEventVisibility::Observable);
655 }
656
MetadataLoaded(UniquePtr<MediaInfo> aInfo,UniquePtr<MetadataTags> aTags,MediaDecoderEventVisibility aEventVisibility)657 void MediaDecoder::MetadataLoaded(
658 UniquePtr<MediaInfo> aInfo, UniquePtr<MetadataTags> aTags,
659 MediaDecoderEventVisibility aEventVisibility) {
660 MOZ_ASSERT(NS_IsMainThread());
661 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
662
663 LOG("MetadataLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d",
664 aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(),
665 aInfo->HasVideo());
666
667 mMediaSeekable = aInfo->mMediaSeekable;
668 mMediaSeekableOnlyInBufferedRanges =
669 aInfo->mMediaSeekableOnlyInBufferedRanges;
670 mInfo = std::move(aInfo);
671
672 // Make sure the element and the frame (if any) are told about
673 // our new size.
674 if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) {
675 mFiredMetadataLoaded = true;
676 GetOwner()->MetadataLoaded(mInfo.get(), std::move(aTags));
677 }
678 // Invalidate() will end up calling GetOwner()->UpdateMediaSize with the last
679 // dimensions retrieved from the video frame container. The video frame
680 // container contains more up to date dimensions than aInfo.
681 // So we call Invalidate() after calling GetOwner()->MetadataLoaded to ensure
682 // the media element has the latest dimensions.
683 Invalidate();
684
685 EnsureTelemetryReported();
686 }
687
EnsureTelemetryReported()688 void MediaDecoder::EnsureTelemetryReported() {
689 MOZ_ASSERT(NS_IsMainThread());
690
691 if (mTelemetryReported || !mInfo) {
692 // Note: sometimes we get multiple MetadataLoaded calls (for example
693 // for chained ogg). So we ensure we don't report duplicate results for
694 // these resources.
695 return;
696 }
697
698 nsTArray<nsCString> codecs;
699 if (mInfo->HasAudio() &&
700 !mInfo->mAudio.GetAsAudioInfo()->mMimeType.IsEmpty()) {
701 codecs.AppendElement(mInfo->mAudio.GetAsAudioInfo()->mMimeType);
702 }
703 if (mInfo->HasVideo() &&
704 !mInfo->mVideo.GetAsVideoInfo()->mMimeType.IsEmpty()) {
705 codecs.AppendElement(mInfo->mVideo.GetAsVideoInfo()->mMimeType);
706 }
707 if (codecs.IsEmpty()) {
708 codecs.AppendElement(nsPrintfCString(
709 "resource; %s", ContainerType().OriginalString().Data()));
710 }
711 for (const nsCString& codec : codecs) {
712 LOG("Telemetry MEDIA_CODEC_USED= '%s'", codec.get());
713 Telemetry::Accumulate(Telemetry::HistogramID::MEDIA_CODEC_USED, codec);
714 }
715
716 mTelemetryReported = true;
717 }
718
PlayStateStr()719 const char* MediaDecoder::PlayStateStr() {
720 MOZ_ASSERT(NS_IsMainThread());
721 return ToPlayStateStr(mPlayState);
722 }
723
FirstFrameLoaded(UniquePtr<MediaInfo> aInfo,MediaDecoderEventVisibility aEventVisibility)724 void MediaDecoder::FirstFrameLoaded(
725 UniquePtr<MediaInfo> aInfo, MediaDecoderEventVisibility aEventVisibility) {
726 MOZ_ASSERT(NS_IsMainThread());
727 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
728
729 LOG("FirstFrameLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d "
730 "mPlayState=%s transportSeekable=%d",
731 aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(),
732 aInfo->HasVideo(), PlayStateStr(), IsTransportSeekable());
733
734 mInfo = std::move(aInfo);
735
736 Invalidate();
737
738 // The element can run javascript via events
739 // before reaching here, so only change the
740 // state if we're still set to the original
741 // loading state.
742 if (mPlayState == PLAY_STATE_LOADING) {
743 ChangeState(mNextState);
744 }
745
746 // GetOwner()->FirstFrameLoaded() might call us back. Put it at the bottom of
747 // this function to avoid unexpected shutdown from reentrant calls.
748 if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) {
749 GetOwner()->FirstFrameLoaded();
750 }
751 }
752
NetworkError(const MediaResult & aError)753 void MediaDecoder::NetworkError(const MediaResult& aError) {
754 MOZ_ASSERT(NS_IsMainThread());
755 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
756 GetOwner()->NetworkError(aError);
757 }
758
DecodeError(const MediaResult & aError)759 void MediaDecoder::DecodeError(const MediaResult& aError) {
760 MOZ_ASSERT(NS_IsMainThread());
761 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
762 GetOwner()->DecodeError(aError);
763 }
764
UpdateSameOriginStatus(bool aSameOrigin)765 void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) {
766 MOZ_ASSERT(NS_IsMainThread());
767 mSameOriginMedia = aSameOrigin;
768 }
769
IsSeeking() const770 bool MediaDecoder::IsSeeking() const {
771 MOZ_ASSERT(NS_IsMainThread());
772 return mLogicallySeeking;
773 }
774
OwnerHasError() const775 bool MediaDecoder::OwnerHasError() const {
776 MOZ_ASSERT(NS_IsMainThread());
777 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
778 return GetOwner()->HasError();
779 }
780
IsEnded() const781 bool MediaDecoder::IsEnded() const {
782 MOZ_ASSERT(NS_IsMainThread());
783 return mPlayState == PLAY_STATE_ENDED;
784 }
785
IsShutdown() const786 bool MediaDecoder::IsShutdown() const {
787 MOZ_ASSERT(NS_IsMainThread());
788 return mPlayState == PLAY_STATE_SHUTDOWN;
789 }
790
PlaybackEnded()791 void MediaDecoder::PlaybackEnded() {
792 MOZ_ASSERT(NS_IsMainThread());
793 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
794
795 if (mLogicallySeeking || mPlayState == PLAY_STATE_LOADING ||
796 mPlayState == PLAY_STATE_ENDED) {
797 LOG("MediaDecoder::PlaybackEnded bailed out, "
798 "mLogicallySeeking=%d mPlayState=%s",
799 mLogicallySeeking.Ref(), ToPlayStateStr(mPlayState));
800 return;
801 }
802
803 LOG("MediaDecoder::PlaybackEnded");
804
805 ChangeState(PLAY_STATE_ENDED);
806 InvalidateWithFlags(VideoFrameContainer::INVALIDATE_FORCE);
807 GetOwner()->PlaybackEnded();
808 }
809
NotifyPrincipalChanged()810 void MediaDecoder::NotifyPrincipalChanged() {
811 MOZ_ASSERT(NS_IsMainThread());
812 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
813 GetOwner()->NotifyDecoderPrincipalChanged();
814 }
815
OnSeekResolved()816 void MediaDecoder::OnSeekResolved() {
817 MOZ_ASSERT(NS_IsMainThread());
818 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
819 mLogicallySeeking = false;
820
821 // Ensure logical position is updated after seek.
822 UpdateLogicalPositionInternal();
823 mSeekRequest.Complete();
824
825 GetOwner()->SeekCompleted();
826 }
827
OnSeekRejected()828 void MediaDecoder::OnSeekRejected() {
829 MOZ_ASSERT(NS_IsMainThread());
830 mSeekRequest.Complete();
831 mLogicallySeeking = false;
832
833 GetOwner()->SeekAborted();
834 }
835
SeekingStarted()836 void MediaDecoder::SeekingStarted() {
837 MOZ_ASSERT(NS_IsMainThread());
838 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
839 GetOwner()->SeekStarted();
840 }
841
ChangeState(PlayState aState)842 void MediaDecoder::ChangeState(PlayState aState) {
843 MOZ_ASSERT(NS_IsMainThread());
844 MOZ_ASSERT(!IsShutdown(), "SHUTDOWN is the final state.");
845
846 if (mNextState == aState) {
847 mNextState = PLAY_STATE_PAUSED;
848 }
849
850 if (mPlayState != aState) {
851 DDLOG(DDLogCategory::Property, "play_state", ToPlayStateStr(aState));
852 }
853 LOG("Play state changes from %s to %s", ToPlayStateStr(mPlayState),
854 ToPlayStateStr(aState));
855 mPlayState = aState;
856 UpdateTelemetryHelperBasedOnPlayState(aState);
857 }
858
OwnerVisibility() const859 TelemetryProbesReporter::Visibility MediaDecoder::OwnerVisibility() const {
860 return GetOwner()->IsActuallyInvisible() || mForcedHidden
861 ? TelemetryProbesReporter::Visibility::eInvisible
862 : TelemetryProbesReporter::Visibility::eVisible;
863 }
864
UpdateTelemetryHelperBasedOnPlayState(PlayState aState) const865 void MediaDecoder::UpdateTelemetryHelperBasedOnPlayState(
866 PlayState aState) const {
867 if (aState == PlayState::PLAY_STATE_PLAYING) {
868 mTelemetryProbesReporter->OnPlay(OwnerVisibility());
869 } else if (aState == PlayState::PLAY_STATE_PAUSED ||
870 aState == PlayState::PLAY_STATE_ENDED) {
871 mTelemetryProbesReporter->OnPause(OwnerVisibility());
872 } else if (aState == PLAY_STATE_SHUTDOWN) {
873 mTelemetryProbesReporter->OnShutdown();
874 }
875 }
876
GetPositionUpdateReason(double aPrevPos,double aCurPos) const877 MediaDecoder::PositionUpdate MediaDecoder::GetPositionUpdateReason(
878 double aPrevPos, double aCurPos) const {
879 MOZ_ASSERT(NS_IsMainThread());
880 // If current position is earlier than previous position and we didn't do
881 // seek, that means we looped back to the start position, which currently
882 // happens on audio only.
883 const bool notSeeking = !mSeekRequest.Exists();
884 if (mLooping && notSeeking && aCurPos < aPrevPos) {
885 return PositionUpdate::eSeamlessLoopingSeeking;
886 }
887 return aPrevPos != aCurPos && notSeeking ? PositionUpdate::ePeriodicUpdate
888 : PositionUpdate::eOther;
889 }
890
UpdateLogicalPositionInternal()891 void MediaDecoder::UpdateLogicalPositionInternal() {
892 MOZ_ASSERT(NS_IsMainThread());
893 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
894
895 double currentPosition = CurrentPosition().ToSeconds();
896 if (mPlayState == PLAY_STATE_ENDED) {
897 currentPosition = std::max(currentPosition, mDuration);
898 }
899
900 const PositionUpdate reason =
901 GetPositionUpdateReason(mLogicalPosition, currentPosition);
902 switch (reason) {
903 case PositionUpdate::ePeriodicUpdate:
904 SetLogicalPosition(currentPosition);
905 // This is actually defined in `TimeMarchesOn`, but we do that in decoder.
906 // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:event-media-timeupdate-7
907 // TODO (bug 1688137): should we move it back to `TimeMarchesOn`?
908 GetOwner()->MaybeQueueTimeupdateEvent();
909 break;
910 case PositionUpdate::eSeamlessLoopingSeeking:
911 // When seamless seeking occurs, seeking was performed on the demuxer so
912 // the decoder doesn't know. That means decoder still thinks it's in
913 // playing. Therefore, we have to manually call those methods to notify
914 // the owner about seeking.
915 GetOwner()->SeekStarted();
916 SetLogicalPosition(currentPosition);
917 GetOwner()->SeekCompleted();
918 break;
919 default:
920 MOZ_ASSERT(reason == PositionUpdate::eOther);
921 SetLogicalPosition(currentPosition);
922 break;
923 }
924
925 // Invalidate the frame so any video data is displayed.
926 // Do this before the timeupdate event so that if that
927 // event runs JavaScript that queries the media size, the
928 // frame has reflowed and the size updated beforehand.
929 Invalidate();
930 }
931
SetLogicalPosition(double aNewPosition)932 void MediaDecoder::SetLogicalPosition(double aNewPosition) {
933 MOZ_ASSERT(NS_IsMainThread());
934 if (mLogicalPosition == aNewPosition) {
935 return;
936 }
937 mLogicalPosition = aNewPosition;
938 DDLOG(DDLogCategory::Property, "currentTime", mLogicalPosition);
939 }
940
DurationChanged()941 void MediaDecoder::DurationChanged() {
942 MOZ_ASSERT(NS_IsMainThread());
943 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
944
945 double oldDuration = mDuration;
946
947 // Use the explicit duration if we have one.
948 // Otherwise use the duration mirrored from MDSM.
949 if (mExplicitDuration.isSome()) {
950 mDuration = mExplicitDuration.ref();
951 } else if (mStateMachineDuration.Ref().isSome()) {
952 mDuration = mStateMachineDuration.Ref().ref().ToSeconds();
953 }
954
955 if (mDuration == oldDuration || IsNaN(mDuration)) {
956 return;
957 }
958
959 LOG("Duration changed to %f", mDuration);
960
961 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=28822 for a discussion
962 // of whether we should fire durationchange on explicit infinity.
963 if (mFiredMetadataLoaded &&
964 (!mozilla::IsInfinite<double>(mDuration) || mExplicitDuration.isSome())) {
965 GetOwner()->DispatchAsyncEvent(u"durationchange"_ns);
966 }
967
968 if (CurrentPosition() > TimeUnit::FromSeconds(mDuration)) {
969 Seek(mDuration, SeekTarget::Accurate);
970 }
971 }
972
GetCompositor()973 already_AddRefed<KnowsCompositor> MediaDecoder::GetCompositor() {
974 MediaDecoderOwner* owner = GetOwner();
975 Document* ownerDoc = owner ? owner->GetDocument() : nullptr;
976 RefPtr<LayerManager> layerManager =
977 ownerDoc ? nsContentUtils::LayerManagerForDocument(ownerDoc) : nullptr;
978 RefPtr<KnowsCompositor> knows =
979 layerManager ? layerManager->AsKnowsCompositor() : nullptr;
980 return knows ? knows->GetForMedia().forget() : nullptr;
981 }
982
NotifyCompositor()983 void MediaDecoder::NotifyCompositor() {
984 RefPtr<KnowsCompositor> knowsCompositor = GetCompositor();
985 if (knowsCompositor) {
986 nsCOMPtr<nsIRunnable> r =
987 NewRunnableMethod<already_AddRefed<KnowsCompositor>&&>(
988 "MediaFormatReader::UpdateCompositor", mReader,
989 &MediaFormatReader::UpdateCompositor, knowsCompositor.forget());
990 Unused << mReader->OwnerThread()->Dispatch(r.forget());
991 }
992 }
993
SetElementVisibility(bool aIsOwnerInvisible,bool aIsOwnerConnected)994 void MediaDecoder::SetElementVisibility(bool aIsOwnerInvisible,
995 bool aIsOwnerConnected) {
996 MOZ_ASSERT(NS_IsMainThread());
997 mIsOwnerInvisible = aIsOwnerInvisible;
998 mIsOwnerConnected = aIsOwnerConnected;
999 mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility());
1000 UpdateVideoDecodeMode();
1001 }
1002
SetForcedHidden(bool aForcedHidden)1003 void MediaDecoder::SetForcedHidden(bool aForcedHidden) {
1004 MOZ_ASSERT(NS_IsMainThread());
1005 mForcedHidden = aForcedHidden;
1006 mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility());
1007 UpdateVideoDecodeMode();
1008 }
1009
SetSuspendTaint(bool aTainted)1010 void MediaDecoder::SetSuspendTaint(bool aTainted) {
1011 MOZ_ASSERT(NS_IsMainThread());
1012 mHasSuspendTaint = aTainted;
1013 UpdateVideoDecodeMode();
1014 }
1015
UpdateVideoDecodeMode()1016 void MediaDecoder::UpdateVideoDecodeMode() {
1017 MOZ_ASSERT(NS_IsMainThread());
1018
1019 // The MDSM may yet be set.
1020 if (!mDecoderStateMachine) {
1021 LOG("UpdateVideoDecodeMode(), early return because we don't have MDSM.");
1022 return;
1023 }
1024
1025 // Seeking is required when leaving suspend mode.
1026 if (!mMediaSeekable) {
1027 LOG("UpdateVideoDecodeMode(), set Normal because the media is not "
1028 "seekable");
1029 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1030 return;
1031 }
1032
1033 // If mHasSuspendTaint is set, never suspend the video decoder.
1034 if (mHasSuspendTaint) {
1035 LOG("UpdateVideoDecodeMode(), set Normal because the element has been "
1036 "tainted.");
1037 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1038 return;
1039 }
1040
1041 // If mSecondaryVideoContainer is set, never suspend the video decoder.
1042 if (mSecondaryVideoContainer.Ref()) {
1043 LOG("UpdateVideoDecodeMode(), set Normal because the element is cloning "
1044 "itself visually to another video container.");
1045 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1046 return;
1047 }
1048
1049 // Don't suspend elements that is not in a connected tree.
1050 if (!mIsOwnerConnected) {
1051 LOG("UpdateVideoDecodeMode(), set Normal because the element is not in "
1052 "tree.");
1053 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1054 return;
1055 }
1056
1057 // If mForcedHidden is set, suspend the video decoder anyway.
1058 if (mForcedHidden) {
1059 LOG("UpdateVideoDecodeMode(), set Suspend because the element is forced to "
1060 "be suspended.");
1061 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend);
1062 return;
1063 }
1064
1065 // Resume decoding in the advance, even the element is in the background.
1066 if (mIsBackgroundVideoDecodingAllowed) {
1067 LOG("UpdateVideoDecodeMode(), set Normal because the tab is in background "
1068 "and hovered.");
1069 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1070 return;
1071 }
1072
1073 if (mIsOwnerInvisible) {
1074 LOG("UpdateVideoDecodeMode(), set Suspend because of invisible element.");
1075 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend);
1076 } else {
1077 LOG("UpdateVideoDecodeMode(), set Normal because of visible element.");
1078 mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1079 }
1080 }
1081
SetIsBackgroundVideoDecodingAllowed(bool aAllowed)1082 void MediaDecoder::SetIsBackgroundVideoDecodingAllowed(bool aAllowed) {
1083 mIsBackgroundVideoDecodingAllowed = aAllowed;
1084 UpdateVideoDecodeMode();
1085 }
1086
HasSuspendTaint() const1087 bool MediaDecoder::HasSuspendTaint() const {
1088 MOZ_ASSERT(NS_IsMainThread());
1089 return mHasSuspendTaint;
1090 }
1091
SetSecondaryVideoContainer(RefPtr<VideoFrameContainer> aSecondaryVideoContainer)1092 void MediaDecoder::SetSecondaryVideoContainer(
1093 RefPtr<VideoFrameContainer> aSecondaryVideoContainer) {
1094 MOZ_ASSERT(NS_IsMainThread());
1095 if (mSecondaryVideoContainer.Ref() == aSecondaryVideoContainer) {
1096 return;
1097 }
1098 mSecondaryVideoContainer = std::move(aSecondaryVideoContainer);
1099 UpdateVideoDecodeMode();
1100 }
1101
IsMediaSeekable()1102 bool MediaDecoder::IsMediaSeekable() {
1103 MOZ_ASSERT(NS_IsMainThread());
1104 NS_ENSURE_TRUE(GetStateMachine(), false);
1105 return mMediaSeekable;
1106 }
1107
GetSeekable()1108 media::TimeIntervals MediaDecoder::GetSeekable() {
1109 MOZ_ASSERT(NS_IsMainThread());
1110
1111 if (IsNaN(GetDuration())) {
1112 // We do not have a duration yet, we can't determine the seekable range.
1113 return TimeIntervals();
1114 }
1115
1116 // We can seek in buffered range if the media is seekable. Also, we can seek
1117 // in unbuffered ranges if the transport level is seekable (local file or the
1118 // server supports range requests, etc.) or in cue-less WebMs
1119 if (mMediaSeekableOnlyInBufferedRanges) {
1120 return GetBuffered();
1121 } else if (!IsMediaSeekable()) {
1122 return media::TimeIntervals();
1123 } else if (!IsTransportSeekable()) {
1124 return GetBuffered();
1125 } else {
1126 return media::TimeIntervals(media::TimeInterval(
1127 TimeUnit::Zero(), IsInfinite() ? TimeUnit::FromInfinity()
1128 : TimeUnit::FromSeconds(GetDuration())));
1129 }
1130 }
1131
SetFragmentEndTime(double aTime)1132 void MediaDecoder::SetFragmentEndTime(double aTime) {
1133 MOZ_ASSERT(NS_IsMainThread());
1134 if (mDecoderStateMachine) {
1135 mDecoderStateMachine->DispatchSetFragmentEndTime(
1136 TimeUnit::FromSeconds(aTime));
1137 }
1138 }
1139
SetPlaybackRate(double aPlaybackRate)1140 void MediaDecoder::SetPlaybackRate(double aPlaybackRate) {
1141 MOZ_ASSERT(NS_IsMainThread());
1142
1143 double oldRate = mPlaybackRate;
1144 mPlaybackRate = aPlaybackRate;
1145 if (aPlaybackRate == 0) {
1146 Pause();
1147 return;
1148 }
1149
1150 if (oldRate == 0 && !GetOwner()->GetPaused()) {
1151 // PlaybackRate is no longer null.
1152 // Restart the playback if the media was playing.
1153 Play();
1154 }
1155
1156 if (mDecoderStateMachine) {
1157 mDecoderStateMachine->DispatchSetPlaybackRate(aPlaybackRate);
1158 }
1159 }
1160
SetPreservesPitch(bool aPreservesPitch)1161 void MediaDecoder::SetPreservesPitch(bool aPreservesPitch) {
1162 MOZ_ASSERT(NS_IsMainThread());
1163 mPreservesPitch = aPreservesPitch;
1164 }
1165
SetLooping(bool aLooping)1166 void MediaDecoder::SetLooping(bool aLooping) {
1167 MOZ_ASSERT(NS_IsMainThread());
1168 mLooping = aLooping;
1169 }
1170
SetStreamName(const nsAutoString & aStreamName)1171 void MediaDecoder::SetStreamName(const nsAutoString& aStreamName) {
1172 MOZ_ASSERT(NS_IsMainThread());
1173 mStreamName = aStreamName;
1174 }
1175
ConnectMirrors(MediaDecoderStateMachine * aObject)1176 void MediaDecoder::ConnectMirrors(MediaDecoderStateMachine* aObject) {
1177 MOZ_ASSERT(NS_IsMainThread());
1178 MOZ_ASSERT(aObject);
1179 mStateMachineDuration.Connect(aObject->CanonicalDuration());
1180 mBuffered.Connect(aObject->CanonicalBuffered());
1181 mCurrentPosition.Connect(aObject->CanonicalCurrentPosition());
1182 mIsAudioDataAudible.Connect(aObject->CanonicalIsAudioDataAudible());
1183 }
1184
DisconnectMirrors()1185 void MediaDecoder::DisconnectMirrors() {
1186 MOZ_ASSERT(NS_IsMainThread());
1187 mStateMachineDuration.DisconnectIfConnected();
1188 mBuffered.DisconnectIfConnected();
1189 mCurrentPosition.DisconnectIfConnected();
1190 mIsAudioDataAudible.DisconnectIfConnected();
1191 }
1192
SetStateMachine(MediaDecoderStateMachine * aStateMachine)1193 void MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine) {
1194 MOZ_ASSERT(NS_IsMainThread());
1195 MOZ_ASSERT_IF(aStateMachine, !mDecoderStateMachine);
1196 if (aStateMachine) {
1197 mDecoderStateMachine = aStateMachine;
1198 DDLINKCHILD("decoder state machine", mDecoderStateMachine.get());
1199 ConnectMirrors(aStateMachine);
1200 UpdateVideoDecodeMode();
1201 } else if (mDecoderStateMachine) {
1202 DDUNLINKCHILD(mDecoderStateMachine.get());
1203 mDecoderStateMachine = nullptr;
1204 DisconnectMirrors();
1205 }
1206 }
1207
GetImageContainer()1208 ImageContainer* MediaDecoder::GetImageContainer() {
1209 return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
1210 : nullptr;
1211 }
1212
InvalidateWithFlags(uint32_t aFlags)1213 void MediaDecoder::InvalidateWithFlags(uint32_t aFlags) {
1214 if (mVideoFrameContainer) {
1215 mVideoFrameContainer->InvalidateWithFlags(aFlags);
1216 }
1217 }
1218
Invalidate()1219 void MediaDecoder::Invalidate() {
1220 if (mVideoFrameContainer) {
1221 mVideoFrameContainer->Invalidate();
1222 }
1223 }
1224
Suspend()1225 void MediaDecoder::Suspend() {
1226 MOZ_ASSERT(NS_IsMainThread());
1227 GetStateMachine()->InvokeSuspendMediaSink();
1228 }
1229
Resume()1230 void MediaDecoder::Resume() {
1231 MOZ_ASSERT(NS_IsMainThread());
1232 GetStateMachine()->InvokeResumeMediaSink();
1233 }
1234
1235 // Constructs the time ranges representing what segments of the media
1236 // are buffered and playable.
GetBuffered()1237 media::TimeIntervals MediaDecoder::GetBuffered() {
1238 MOZ_ASSERT(NS_IsMainThread());
1239 return mBuffered.Ref();
1240 }
1241
SizeOfVideoQueue()1242 size_t MediaDecoder::SizeOfVideoQueue() {
1243 MOZ_ASSERT(NS_IsMainThread());
1244 if (mDecoderStateMachine) {
1245 return mDecoderStateMachine->SizeOfVideoQueue();
1246 }
1247 return 0;
1248 }
1249
SizeOfAudioQueue()1250 size_t MediaDecoder::SizeOfAudioQueue() {
1251 MOZ_ASSERT(NS_IsMainThread());
1252 if (mDecoderStateMachine) {
1253 return mDecoderStateMachine->SizeOfAudioQueue();
1254 }
1255 return 0;
1256 }
1257
NotifyReaderDataArrived()1258 void MediaDecoder::NotifyReaderDataArrived() {
1259 MOZ_ASSERT(NS_IsMainThread());
1260 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1261
1262 nsresult rv = mReader->OwnerThread()->Dispatch(
1263 NewRunnableMethod("MediaFormatReader::NotifyDataArrived", mReader.get(),
1264 &MediaFormatReader::NotifyDataArrived));
1265 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1266 Unused << rv;
1267 }
1268
1269 // Provide access to the state machine object
GetStateMachine() const1270 MediaDecoderStateMachine* MediaDecoder::GetStateMachine() const {
1271 MOZ_ASSERT(NS_IsMainThread());
1272 return mDecoderStateMachine;
1273 }
1274
CanPlayThrough()1275 bool MediaDecoder::CanPlayThrough() {
1276 MOZ_ASSERT(NS_IsMainThread());
1277 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1278 return CanPlayThroughImpl();
1279 }
1280
SetCDMProxy(CDMProxy * aProxy)1281 RefPtr<SetCDMPromise> MediaDecoder::SetCDMProxy(CDMProxy* aProxy) {
1282 MOZ_ASSERT(NS_IsMainThread());
1283 return InvokeAsync<RefPtr<CDMProxy>>(mReader->OwnerThread(), mReader.get(),
1284 __func__,
1285 &MediaFormatReader::SetCDMProxy, aProxy);
1286 }
1287
IsOpusEnabled()1288 bool MediaDecoder::IsOpusEnabled() { return StaticPrefs::media_opus_enabled(); }
1289
IsOggEnabled()1290 bool MediaDecoder::IsOggEnabled() { return StaticPrefs::media_ogg_enabled(); }
1291
IsWaveEnabled()1292 bool MediaDecoder::IsWaveEnabled() { return StaticPrefs::media_wave_enabled(); }
1293
IsWebMEnabled()1294 bool MediaDecoder::IsWebMEnabled() { return StaticPrefs::media_webm_enabled(); }
1295
1296 NS_IMETHODIMP
CollectReports(nsIHandleReportCallback * aHandleReport,nsISupports * aData,bool aAnonymize)1297 MediaMemoryTracker::CollectReports(nsIHandleReportCallback* aHandleReport,
1298 nsISupports* aData, bool aAnonymize) {
1299 // NB: When resourceSizes' ref count goes to 0 the promise will report the
1300 // resources memory and finish the asynchronous memory report.
1301 RefPtr<MediaDecoder::ResourceSizes> resourceSizes =
1302 new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf);
1303
1304 nsCOMPtr<nsIHandleReportCallback> handleReport = aHandleReport;
1305 nsCOMPtr<nsISupports> data = aData;
1306
1307 resourceSizes->Promise()->Then(
1308 AbstractThread::MainThread(), __func__,
1309 [handleReport, data](size_t size) {
1310 handleReport->Callback(
1311 ""_ns, "explicit/media/resources"_ns, KIND_HEAP, UNITS_BYTES, size,
1312 nsLiteralCString("Memory used by media resources including "
1313 "streaming buffers, caches, etc."),
1314 data);
1315
1316 nsCOMPtr<nsIMemoryReporterManager> imgr =
1317 do_GetService("@mozilla.org/memory-reporter-manager;1");
1318
1319 if (imgr) {
1320 imgr->EndReport();
1321 }
1322 },
1323 [](size_t) { /* unused reject function */ });
1324
1325 int64_t video = 0;
1326 int64_t audio = 0;
1327 DecodersArray& decoders = Decoders();
1328 for (size_t i = 0; i < decoders.Length(); ++i) {
1329 MediaDecoder* decoder = decoders[i];
1330 video += decoder->SizeOfVideoQueue();
1331 audio += decoder->SizeOfAudioQueue();
1332 decoder->AddSizeOfResources(resourceSizes);
1333 }
1334
1335 MOZ_COLLECT_REPORT("explicit/media/decoded/video", KIND_HEAP, UNITS_BYTES,
1336 video, "Memory used by decoded video frames.");
1337
1338 MOZ_COLLECT_REPORT("explicit/media/decoded/audio", KIND_HEAP, UNITS_BYTES,
1339 audio, "Memory used by decoded audio chunks.");
1340
1341 return NS_OK;
1342 }
1343
GetOwner() const1344 MediaDecoderOwner* MediaDecoder::GetOwner() const {
1345 MOZ_ASSERT(NS_IsMainThread());
1346 // mOwner is valid until shutdown.
1347 return mOwner;
1348 }
1349
NextFrameBufferedStatus()1350 MediaDecoderOwner::NextFrameStatus MediaDecoder::NextFrameBufferedStatus() {
1351 MOZ_ASSERT(NS_IsMainThread());
1352 // Next frame hasn't been decoded yet.
1353 // Use the buffered range to consider if we have the next frame available.
1354 auto currentPosition = CurrentPosition();
1355 media::TimeInterval interval(
1356 currentPosition, currentPosition + DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED);
1357 return GetBuffered().Contains(interval)
1358 ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
1359 : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
1360 }
1361
GetDebugInfo(dom::MediaDecoderDebugInfo & aInfo)1362 void MediaDecoder::GetDebugInfo(dom::MediaDecoderDebugInfo& aInfo) {
1363 CopyUTF8toUTF16(nsPrintfCString("%p", this), aInfo.mInstance);
1364 aInfo.mChannels = mInfo ? mInfo->mAudio.mChannels : 0;
1365 aInfo.mRate = mInfo ? mInfo->mAudio.mRate : 0;
1366 aInfo.mHasAudio = mInfo ? mInfo->HasAudio() : false;
1367 aInfo.mHasVideo = mInfo ? mInfo->HasVideo() : false;
1368 CopyUTF8toUTF16(MakeStringSpan(PlayStateStr()), aInfo.mPlayState);
1369 aInfo.mContainerType =
1370 NS_ConvertUTF8toUTF16(ContainerType().Type().AsString());
1371 mReader->GetDebugInfo(aInfo.mReader);
1372 }
1373
RequestDebugInfo(MediaDecoderDebugInfo & aInfo)1374 RefPtr<GenericPromise> MediaDecoder::RequestDebugInfo(
1375 MediaDecoderDebugInfo& aInfo) {
1376 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1377 GetDebugInfo(aInfo);
1378
1379 if (!GetStateMachine()) {
1380 return GenericPromise::CreateAndResolve(true, __func__);
1381 }
1382
1383 return GetStateMachine()
1384 ->RequestDebugInfo(aInfo.mStateMachine)
1385 ->Then(
1386 AbstractThread::MainThread(), __func__,
1387 []() { return GenericPromise::CreateAndResolve(true, __func__); },
1388 []() {
1389 MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
1390 return GenericPromise::CreateAndResolve(false, __func__);
1391 });
1392 }
1393
NotifyAudibleStateChanged()1394 void MediaDecoder::NotifyAudibleStateChanged() {
1395 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1396 GetOwner()->SetAudibleState(mIsAudioDataAudible);
1397 }
1398
GetTotalPlayTimeInSeconds() const1399 double MediaDecoder::GetTotalPlayTimeInSeconds() const {
1400 return mTelemetryProbesReporter->GetTotalPlayTimeInSeconds();
1401 }
1402
GetInvisibleVideoPlayTimeInSeconds() const1403 double MediaDecoder::GetInvisibleVideoPlayTimeInSeconds() const {
1404 return mTelemetryProbesReporter->GetInvisibleVideoPlayTimeInSeconds();
1405 }
1406
GetVideoDecodeSuspendedTimeInSeconds() const1407 double MediaDecoder::GetVideoDecodeSuspendedTimeInSeconds() const {
1408 return mTelemetryProbesReporter->GetVideoDecodeSuspendedTimeInSeconds();
1409 }
1410
1411 MediaMemoryTracker::MediaMemoryTracker() = default;
1412
InitMemoryReporter()1413 void MediaMemoryTracker::InitMemoryReporter() {
1414 RegisterWeakAsyncMemoryReporter(this);
1415 }
1416
~MediaMemoryTracker()1417 MediaMemoryTracker::~MediaMemoryTracker() {
1418 UnregisterWeakMemoryReporter(this);
1419 }
1420
1421 } // namespace mozilla
1422
1423 // avoid redefined macro in unified build
1424 #undef DUMP
1425 #undef LOG
1426 #undef NS_DispatchToMainThread
1427