1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #include "MediaDecoder.h"
8 
9 #include "AudioDeviceInfo.h"
10 #include "DOMMediaStream.h"
11 #include "DecoderBenchmark.h"
12 #include "ImageContainer.h"
13 #include "Layers.h"
14 #include "MediaDecoderStateMachine.h"
15 #include "MediaFormatReader.h"
16 #include "MediaResource.h"
17 #include "MediaShutdownManager.h"
18 #include "MediaTrackGraph.h"
19 #include "TelemetryProbesReporter.h"
20 #include "VideoFrameContainer.h"
21 #include "VideoUtils.h"
22 #include "mozilla/AbstractThread.h"
23 #include "mozilla/dom/DOMTypes.h"
24 #include "mozilla/FloatingPoint.h"
25 #include "mozilla/MathAlgorithms.h"
26 #include "mozilla/Preferences.h"
27 #include "mozilla/StaticPrefs_media.h"
28 #include "mozilla/StaticPtr.h"
29 #include "mozilla/Telemetry.h"
30 #include "mozilla/Unused.h"
31 #include "nsComponentManagerUtils.h"
32 #include "nsContentUtils.h"
33 #include "nsError.h"
34 #include "nsIMemoryReporter.h"
35 #include "nsPrintfCString.h"
36 #include "nsServiceManagerUtils.h"
37 #include "nsTArray.h"
38 #include <algorithm>
39 #include <limits>
40 
41 using namespace mozilla::dom;
42 using namespace mozilla::layers;
43 using namespace mozilla::media;
44 
45 namespace mozilla {
46 
47 // avoid redefined macro in unified build
48 #undef LOG
49 #undef DUMP
50 
51 LazyLogModule gMediaDecoderLog("MediaDecoder");
52 #define LOG(x, ...) \
53   DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, x, ##__VA_ARGS__)
54 
55 #define DUMP(x, ...) printf_stderr(x "\n", ##__VA_ARGS__)
56 
57 #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
58 
ToPlayStateStr(MediaDecoder::PlayState aState)59 static const char* ToPlayStateStr(MediaDecoder::PlayState aState) {
60   switch (aState) {
61     case MediaDecoder::PLAY_STATE_LOADING:
62       return "LOADING";
63     case MediaDecoder::PLAY_STATE_PAUSED:
64       return "PAUSED";
65     case MediaDecoder::PLAY_STATE_PLAYING:
66       return "PLAYING";
67     case MediaDecoder::PLAY_STATE_ENDED:
68       return "ENDED";
69     case MediaDecoder::PLAY_STATE_SHUTDOWN:
70       return "SHUTDOWN";
71     default:
72       MOZ_ASSERT_UNREACHABLE("Invalid playState.");
73   }
74   return "UNKNOWN";
75 }
76 
77 class MediaMemoryTracker : public nsIMemoryReporter {
78   virtual ~MediaMemoryTracker();
79 
80   NS_DECL_THREADSAFE_ISUPPORTS
81   NS_DECL_NSIMEMORYREPORTER
82 
83   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
84 
85   MediaMemoryTracker();
86   void InitMemoryReporter();
87 
88   static StaticRefPtr<MediaMemoryTracker> sUniqueInstance;
89 
UniqueInstance()90   static MediaMemoryTracker* UniqueInstance() {
91     if (!sUniqueInstance) {
92       sUniqueInstance = new MediaMemoryTracker();
93       sUniqueInstance->InitMemoryReporter();
94     }
95     return sUniqueInstance;
96   }
97 
98   typedef nsTArray<MediaDecoder*> DecodersArray;
Decoders()99   static DecodersArray& Decoders() { return UniqueInstance()->mDecoders; }
100 
101   DecodersArray mDecoders;
102 
103  public:
AddMediaDecoder(MediaDecoder * aDecoder)104   static void AddMediaDecoder(MediaDecoder* aDecoder) {
105     Decoders().AppendElement(aDecoder);
106   }
107 
RemoveMediaDecoder(MediaDecoder * aDecoder)108   static void RemoveMediaDecoder(MediaDecoder* aDecoder) {
109     DecodersArray& decoders = Decoders();
110     decoders.RemoveElement(aDecoder);
111     if (decoders.IsEmpty()) {
112       sUniqueInstance = nullptr;
113     }
114   }
115 
GetSizes()116   static RefPtr<MediaMemoryPromise> GetSizes() {
117     MOZ_ASSERT(NS_IsMainThread());
118     DecodersArray& decoders = Decoders();
119 
120     // if we don't have any decoder, we can bail
121     if (decoders.IsEmpty()) {
122       // and release the instance that was created by calling Decoders()
123       sUniqueInstance = nullptr;
124       return MediaMemoryPromise::CreateAndResolve(MediaMemoryInfo(), __func__);
125     }
126 
127     RefPtr<MediaDecoder::ResourceSizes> resourceSizes =
128         new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf);
129 
130     size_t videoSize = 0;
131     size_t audioSize = 0;
132 
133     for (auto&& decoder : decoders) {
134       videoSize += decoder->SizeOfVideoQueue();
135       audioSize += decoder->SizeOfAudioQueue();
136       decoder->AddSizeOfResources(resourceSizes);
137     }
138 
139     return resourceSizes->Promise()->Then(
140         AbstractThread::MainThread(), __func__,
141         [videoSize, audioSize](size_t resourceSize) {
142           return MediaMemoryPromise::CreateAndResolve(
143               MediaMemoryInfo(videoSize, audioSize, resourceSize), __func__);
144         },
145         [](size_t) {
146           return MediaMemoryPromise::CreateAndReject(NS_ERROR_FAILURE,
147                                                      __func__);
148         });
149   }
150 };
151 
152 StaticRefPtr<MediaMemoryTracker> MediaMemoryTracker::sUniqueInstance;
153 
GetMediaMemorySizes()154 RefPtr<MediaMemoryPromise> GetMediaMemorySizes() {
155   return MediaMemoryTracker::GetSizes();
156 }
157 
158 LazyLogModule gMediaTimerLog("MediaTimer");
159 
160 constexpr TimeUnit MediaDecoder::DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED;
161 
InitStatics()162 void MediaDecoder::InitStatics() {
163   MOZ_ASSERT(NS_IsMainThread());
164   // Eagerly init gMediaDecoderLog to work around bug 1415441.
165   MOZ_LOG(gMediaDecoderLog, LogLevel::Info, ("MediaDecoder::InitStatics"));
166 }
167 
NS_IMPL_ISUPPORTS(MediaMemoryTracker,nsIMemoryReporter)168 NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter)
169 
170 void MediaDecoder::NotifyOwnerActivityChanged(bool aIsOwnerInvisible,
171                                               bool aIsOwnerConnected) {
172   MOZ_ASSERT(NS_IsMainThread());
173   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
174   SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected);
175 
176   NotifyCompositor();
177 }
178 
Pause()179 void MediaDecoder::Pause() {
180   MOZ_ASSERT(NS_IsMainThread());
181   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
182   LOG("Pause");
183   if (mPlayState == PLAY_STATE_LOADING || IsEnded()) {
184     mNextState = PLAY_STATE_PAUSED;
185     return;
186   }
187   ChangeState(PLAY_STATE_PAUSED);
188 }
189 
SetVolume(double aVolume)190 void MediaDecoder::SetVolume(double aVolume) {
191   MOZ_ASSERT(NS_IsMainThread());
192   mVolume = aVolume;
193 }
194 
SetSink(AudioDeviceInfo * aSinkDevice)195 RefPtr<GenericPromise> MediaDecoder::SetSink(AudioDeviceInfo* aSinkDevice) {
196   MOZ_ASSERT(NS_IsMainThread());
197   mSinkDevice = aSinkDevice;
198   return GetStateMachine()->InvokeSetSink(aSinkDevice);
199 }
200 
SetOutputCaptureState(OutputCaptureState aState,SharedDummyTrack * aDummyTrack)201 void MediaDecoder::SetOutputCaptureState(OutputCaptureState aState,
202                                          SharedDummyTrack* aDummyTrack) {
203   MOZ_ASSERT(NS_IsMainThread());
204   MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
205   MOZ_ASSERT_IF(aState == OutputCaptureState::Capture, aDummyTrack);
206   mOutputCaptureState = aState;
207   if (mOutputDummyTrack.Ref().get() != aDummyTrack) {
208     mOutputDummyTrack = nsMainThreadPtrHandle<SharedDummyTrack>(
209         MakeAndAddRef<nsMainThreadPtrHolder<SharedDummyTrack>>(
210             "MediaDecoder::mOutputDummyTrack", aDummyTrack));
211   }
212 }
213 
AddOutputTrack(RefPtr<ProcessedMediaTrack> aTrack)214 void MediaDecoder::AddOutputTrack(RefPtr<ProcessedMediaTrack> aTrack) {
215   MOZ_ASSERT(NS_IsMainThread());
216   MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
217   CopyableTArray<RefPtr<ProcessedMediaTrack>> tracks = mOutputTracks;
218   tracks.AppendElement(std::move(aTrack));
219   mOutputTracks = tracks;
220 }
221 
RemoveOutputTrack(const RefPtr<ProcessedMediaTrack> & aTrack)222 void MediaDecoder::RemoveOutputTrack(
223     const RefPtr<ProcessedMediaTrack>& aTrack) {
224   MOZ_ASSERT(NS_IsMainThread());
225   MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
226   CopyableTArray<RefPtr<ProcessedMediaTrack>> tracks = mOutputTracks;
227   if (tracks.RemoveElement(aTrack)) {
228     mOutputTracks = tracks;
229   }
230 }
231 
SetOutputTracksPrincipal(const RefPtr<nsIPrincipal> & aPrincipal)232 void MediaDecoder::SetOutputTracksPrincipal(
233     const RefPtr<nsIPrincipal>& aPrincipal) {
234   MOZ_ASSERT(NS_IsMainThread());
235   MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
236   mOutputPrincipal = MakePrincipalHandle(aPrincipal);
237 }
238 
GetDuration()239 double MediaDecoder::GetDuration() {
240   MOZ_ASSERT(NS_IsMainThread());
241   return mDuration;
242 }
243 
IsInfinite() const244 bool MediaDecoder::IsInfinite() const {
245   MOZ_ASSERT(NS_IsMainThread());
246   return mozilla::IsInfinite<double>(mDuration);
247 }
248 
249 #define INIT_MIRROR(name, val) \
250   name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Mirror)")
251 #define INIT_CANONICAL(name, val) \
252   name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Canonical)")
253 
MediaDecoder(MediaDecoderInit & aInit)254 MediaDecoder::MediaDecoder(MediaDecoderInit& aInit)
255     : mWatchManager(this, aInit.mOwner->AbstractMainThread()),
256       mLogicalPosition(0.0),
257       mDuration(std::numeric_limits<double>::quiet_NaN()),
258       mOwner(aInit.mOwner),
259       mAbstractMainThread(aInit.mOwner->AbstractMainThread()),
260       mFrameStats(new FrameStatistics()),
261       mDecoderBenchmark(new DecoderBenchmark()),
262       mVideoFrameContainer(aInit.mOwner->GetVideoFrameContainer()),
263       mMinimizePreroll(aInit.mMinimizePreroll),
264       mFiredMetadataLoaded(false),
265       mIsOwnerInvisible(false),
266       mIsOwnerConnected(false),
267       mForcedHidden(false),
268       mHasSuspendTaint(aInit.mHasSuspendTaint),
269       mPlaybackRate(aInit.mPlaybackRate),
270       mLogicallySeeking(false, "MediaDecoder::mLogicallySeeking"),
271       INIT_MIRROR(mBuffered, TimeIntervals()),
272       INIT_MIRROR(mCurrentPosition, TimeUnit::Zero()),
273       INIT_MIRROR(mStateMachineDuration, NullableTimeUnit()),
274       INIT_MIRROR(mIsAudioDataAudible, false),
275       INIT_CANONICAL(mVolume, aInit.mVolume),
276       INIT_CANONICAL(mPreservesPitch, aInit.mPreservesPitch),
277       INIT_CANONICAL(mLooping, aInit.mLooping),
278       INIT_CANONICAL(mStreamName, aInit.mStreamName),
279       INIT_CANONICAL(mSinkDevice, nullptr),
280       INIT_CANONICAL(mSecondaryVideoContainer, nullptr),
281       INIT_CANONICAL(mOutputCaptureState, OutputCaptureState::None),
282       INIT_CANONICAL(mOutputDummyTrack, nullptr),
283       INIT_CANONICAL(mOutputTracks, nsTArray<RefPtr<ProcessedMediaTrack>>()),
284       INIT_CANONICAL(mOutputPrincipal, PRINCIPAL_HANDLE_NONE),
285       INIT_CANONICAL(mPlayState, PLAY_STATE_LOADING),
286       mSameOriginMedia(false),
287       mVideoDecodingOberver(
288           new BackgroundVideoDecodingPermissionObserver(this)),
289       mIsBackgroundVideoDecodingAllowed(false),
290       mTelemetryReported(false),
291       mContainerType(aInit.mContainerType),
292       mTelemetryProbesReporter(
293           new TelemetryProbesReporter(aInit.mReporterOwner)) {
294   MOZ_ASSERT(NS_IsMainThread());
295   MOZ_ASSERT(mAbstractMainThread);
296   MediaMemoryTracker::AddMediaDecoder(this);
297 
298   //
299   // Initialize watchers.
300   //
301 
302   // mDuration
303   mWatchManager.Watch(mStateMachineDuration, &MediaDecoder::DurationChanged);
304 
305   // readyState
306   mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateReadyState);
307   // ReadyState computation depends on MediaDecoder::CanPlayThrough, which
308   // depends on the download rate.
309   mWatchManager.Watch(mBuffered, &MediaDecoder::UpdateReadyState);
310 
311   // mLogicalPosition
312   mWatchManager.Watch(mCurrentPosition, &MediaDecoder::UpdateLogicalPosition);
313   mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateLogicalPosition);
314   mWatchManager.Watch(mLogicallySeeking, &MediaDecoder::UpdateLogicalPosition);
315 
316   mWatchManager.Watch(mIsAudioDataAudible,
317                       &MediaDecoder::NotifyAudibleStateChanged);
318 
319   mVideoDecodingOberver->RegisterEvent();
320 }
321 
322 #undef INIT_MIRROR
323 #undef INIT_CANONICAL
324 
Shutdown()325 void MediaDecoder::Shutdown() {
326   MOZ_ASSERT(NS_IsMainThread());
327   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
328 
329   // Unwatch all watch targets to prevent further notifications.
330   mWatchManager.Shutdown();
331 
332   DiscardOngoingSeekIfExists();
333 
334   // This changes the decoder state to SHUTDOWN and does other things
335   // necessary to unblock the state machine thread if it's blocked, so
336   // the asynchronous shutdown in nsDestroyStateMachine won't deadlock.
337   if (mDecoderStateMachine) {
338     mTimedMetadataListener.Disconnect();
339     mMetadataLoadedListener.Disconnect();
340     mFirstFrameLoadedListener.Disconnect();
341     mOnPlaybackEvent.Disconnect();
342     mOnPlaybackErrorEvent.Disconnect();
343     mOnDecoderDoctorEvent.Disconnect();
344     mOnMediaNotSeekable.Disconnect();
345     mOnEncrypted.Disconnect();
346     mOnWaitingForKey.Disconnect();
347     mOnDecodeWarning.Disconnect();
348     mOnNextFrameStatus.Disconnect();
349     mOnSecondaryVideoContainerInstalled.Disconnect();
350     mOnStoreDecoderBenchmark.Disconnect();
351 
352     mDecoderStateMachine->BeginShutdown()->Then(
353         mAbstractMainThread, __func__, this, &MediaDecoder::FinishShutdown,
354         &MediaDecoder::FinishShutdown);
355   } else {
356     // Ensure we always unregister asynchronously in order not to disrupt
357     // the hashtable iterating in MediaShutdownManager::Shutdown().
358     RefPtr<MediaDecoder> self = this;
359     nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
360         "MediaDecoder::Shutdown", [self]() { self->ShutdownInternal(); });
361     mAbstractMainThread->Dispatch(r.forget());
362   }
363 
364   ChangeState(PLAY_STATE_SHUTDOWN);
365   mVideoDecodingOberver->UnregisterEvent();
366   mVideoDecodingOberver = nullptr;
367   mOwner = nullptr;
368 }
369 
NotifyXPCOMShutdown()370 void MediaDecoder::NotifyXPCOMShutdown() {
371   MOZ_ASSERT(NS_IsMainThread());
372   // NotifyXPCOMShutdown will clear its reference to mDecoder. So we must ensure
373   // that this MediaDecoder stays alive until completion.
374   RefPtr<MediaDecoder> kungFuDeathGrip = this;
375   if (auto owner = GetOwner()) {
376     owner->NotifyXPCOMShutdown();
377   } else if (!IsShutdown()) {
378     Shutdown();
379   }
380   MOZ_DIAGNOSTIC_ASSERT(IsShutdown());
381 }
382 
~MediaDecoder()383 MediaDecoder::~MediaDecoder() {
384   MOZ_ASSERT(NS_IsMainThread());
385   MOZ_DIAGNOSTIC_ASSERT(IsShutdown());
386   MediaMemoryTracker::RemoveMediaDecoder(this);
387 }
388 
OnPlaybackEvent(MediaPlaybackEvent && aEvent)389 void MediaDecoder::OnPlaybackEvent(MediaPlaybackEvent&& aEvent) {
390   switch (aEvent.mType) {
391     case MediaPlaybackEvent::PlaybackEnded:
392       PlaybackEnded();
393       break;
394     case MediaPlaybackEvent::SeekStarted:
395       SeekingStarted();
396       break;
397     case MediaPlaybackEvent::Invalidate:
398       Invalidate();
399       break;
400     case MediaPlaybackEvent::EnterVideoSuspend:
401       GetOwner()->DispatchAsyncEvent(u"mozentervideosuspend"_ns);
402       mTelemetryProbesReporter->OnDecodeSuspended();
403       mIsVideoDecodingSuspended = true;
404       break;
405     case MediaPlaybackEvent::ExitVideoSuspend:
406       GetOwner()->DispatchAsyncEvent(u"mozexitvideosuspend"_ns);
407       mTelemetryProbesReporter->OnDecodeResumed();
408       mIsVideoDecodingSuspended = false;
409       break;
410     case MediaPlaybackEvent::StartVideoSuspendTimer:
411       GetOwner()->DispatchAsyncEvent(u"mozstartvideosuspendtimer"_ns);
412       break;
413     case MediaPlaybackEvent::CancelVideoSuspendTimer:
414       GetOwner()->DispatchAsyncEvent(u"mozcancelvideosuspendtimer"_ns);
415       break;
416     case MediaPlaybackEvent::VideoOnlySeekBegin:
417       GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekbegin"_ns);
418       break;
419     case MediaPlaybackEvent::VideoOnlySeekCompleted:
420       GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekcompleted"_ns);
421       break;
422     default:
423       break;
424   }
425 }
426 
IsVideoDecodingSuspended() const427 bool MediaDecoder::IsVideoDecodingSuspended() const {
428   return mIsVideoDecodingSuspended;
429 }
430 
OnPlaybackErrorEvent(const MediaResult & aError)431 void MediaDecoder::OnPlaybackErrorEvent(const MediaResult& aError) {
432   DecodeError(aError);
433 }
434 
OnDecoderDoctorEvent(DecoderDoctorEvent aEvent)435 void MediaDecoder::OnDecoderDoctorEvent(DecoderDoctorEvent aEvent) {
436   MOZ_ASSERT(NS_IsMainThread());
437   // OnDecoderDoctorEvent is disconnected at shutdown time.
438   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
439   Document* doc = GetOwner()->GetDocument();
440   if (!doc) {
441     return;
442   }
443   DecoderDoctorDiagnostics diags;
444   diags.StoreEvent(doc, aEvent, __func__);
445 }
446 
NextFrameStatusToStr(MediaDecoderOwner::NextFrameStatus aStatus)447 static const char* NextFrameStatusToStr(
448     MediaDecoderOwner::NextFrameStatus aStatus) {
449   switch (aStatus) {
450     case MediaDecoderOwner::NEXT_FRAME_AVAILABLE:
451       return "NEXT_FRAME_AVAILABLE";
452     case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE:
453       return "NEXT_FRAME_UNAVAILABLE";
454     case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING:
455       return "NEXT_FRAME_UNAVAILABLE_BUFFERING";
456     case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING:
457       return "NEXT_FRAME_UNAVAILABLE_SEEKING";
458     case MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED:
459       return "NEXT_FRAME_UNINITIALIZED";
460   }
461   return "UNKNOWN";
462 }
463 
OnNextFrameStatus(MediaDecoderOwner::NextFrameStatus aStatus)464 void MediaDecoder::OnNextFrameStatus(
465     MediaDecoderOwner::NextFrameStatus aStatus) {
466   MOZ_ASSERT(NS_IsMainThread());
467   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
468   if (mNextFrameStatus != aStatus) {
469     LOG("Changed mNextFrameStatus to %s", NextFrameStatusToStr(aStatus));
470     mNextFrameStatus = aStatus;
471     UpdateReadyState();
472   }
473 }
474 
OnSecondaryVideoContainerInstalled(const RefPtr<VideoFrameContainer> & aSecondaryVideoContainer)475 void MediaDecoder::OnSecondaryVideoContainerInstalled(
476     const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer) {
477   MOZ_ASSERT(NS_IsMainThread());
478   GetOwner()->OnSecondaryVideoContainerInstalled(aSecondaryVideoContainer);
479 }
480 
OnStoreDecoderBenchmark(const VideoInfo & aInfo)481 void MediaDecoder::OnStoreDecoderBenchmark(const VideoInfo& aInfo) {
482   MOZ_ASSERT(NS_IsMainThread());
483 
484   int32_t videoFrameRate = aInfo.GetFrameRate().ref();
485 
486   if (mFrameStats && videoFrameRate) {
487     DecoderBenchmarkInfo benchmarkInfo{
488         aInfo.mMimeType,
489         aInfo.mDisplay.width,
490         aInfo.mDisplay.height,
491         videoFrameRate,
492         BitDepthForColorDepth(aInfo.mColorDepth),
493     };
494 
495     LOG("Store benchmark: Video width=%d, height=%d, frameRate=%d, content "
496         "type = %s\n",
497         benchmarkInfo.mWidth, benchmarkInfo.mHeight, benchmarkInfo.mFrameRate,
498         benchmarkInfo.mContentType.BeginReading());
499 
500     mDecoderBenchmark->Store(benchmarkInfo, mFrameStats);
501   }
502 }
503 
ShutdownInternal()504 void MediaDecoder::ShutdownInternal() {
505   MOZ_ASSERT(NS_IsMainThread());
506   mVideoFrameContainer = nullptr;
507   mSecondaryVideoContainer = nullptr;
508   MediaShutdownManager::Instance().Unregister(this);
509 }
510 
FinishShutdown()511 void MediaDecoder::FinishShutdown() {
512   MOZ_ASSERT(NS_IsMainThread());
513   SetStateMachine(nullptr);
514   ShutdownInternal();
515 }
516 
InitializeStateMachine()517 nsresult MediaDecoder::InitializeStateMachine() {
518   MOZ_ASSERT(NS_IsMainThread());
519   NS_ASSERTION(mDecoderStateMachine, "Cannot initialize null state machine!");
520 
521   nsresult rv = mDecoderStateMachine->Init(this);
522   NS_ENSURE_SUCCESS(rv, rv);
523 
524   // If some parameters got set before the state machine got created,
525   // set them now
526   SetStateMachineParameters();
527 
528   return NS_OK;
529 }
530 
SetStateMachineParameters()531 void MediaDecoder::SetStateMachineParameters() {
532   MOZ_ASSERT(NS_IsMainThread());
533   if (mPlaybackRate != 1 && mPlaybackRate != 0) {
534     mDecoderStateMachine->DispatchSetPlaybackRate(mPlaybackRate);
535   }
536   mTimedMetadataListener = mDecoderStateMachine->TimedMetadataEvent().Connect(
537       mAbstractMainThread, this, &MediaDecoder::OnMetadataUpdate);
538   mMetadataLoadedListener = mDecoderStateMachine->MetadataLoadedEvent().Connect(
539       mAbstractMainThread, this, &MediaDecoder::MetadataLoaded);
540   mFirstFrameLoadedListener =
541       mDecoderStateMachine->FirstFrameLoadedEvent().Connect(
542           mAbstractMainThread, this, &MediaDecoder::FirstFrameLoaded);
543 
544   mOnPlaybackEvent = mDecoderStateMachine->OnPlaybackEvent().Connect(
545       mAbstractMainThread, this, &MediaDecoder::OnPlaybackEvent);
546   mOnPlaybackErrorEvent = mDecoderStateMachine->OnPlaybackErrorEvent().Connect(
547       mAbstractMainThread, this, &MediaDecoder::OnPlaybackErrorEvent);
548   mOnDecoderDoctorEvent = mDecoderStateMachine->OnDecoderDoctorEvent().Connect(
549       mAbstractMainThread, this, &MediaDecoder::OnDecoderDoctorEvent);
550   mOnMediaNotSeekable = mDecoderStateMachine->OnMediaNotSeekable().Connect(
551       mAbstractMainThread, this, &MediaDecoder::OnMediaNotSeekable);
552   mOnNextFrameStatus = mDecoderStateMachine->OnNextFrameStatus().Connect(
553       mAbstractMainThread, this, &MediaDecoder::OnNextFrameStatus);
554   mOnSecondaryVideoContainerInstalled =
555       mDecoderStateMachine->OnSecondaryVideoContainerInstalled().Connect(
556           mAbstractMainThread, this,
557           &MediaDecoder::OnSecondaryVideoContainerInstalled);
558   mOnStoreDecoderBenchmark = mReader->OnStoreDecoderBenchmark().Connect(
559       mAbstractMainThread, this, &MediaDecoder::OnStoreDecoderBenchmark);
560 
561   mOnEncrypted = mReader->OnEncrypted().Connect(
562       mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DispatchEncrypted);
563   mOnWaitingForKey = mReader->OnWaitingForKey().Connect(
564       mAbstractMainThread, GetOwner(), &MediaDecoderOwner::NotifyWaitingForKey);
565   mOnDecodeWarning = mReader->OnDecodeWarning().Connect(
566       mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DecodeWarning);
567 }
568 
Play()569 void MediaDecoder::Play() {
570   MOZ_ASSERT(NS_IsMainThread());
571 
572   NS_ASSERTION(mDecoderStateMachine != nullptr, "Should have state machine.");
573   LOG("Play");
574   if (mPlaybackRate == 0) {
575     return;
576   }
577 
578   if (IsEnded()) {
579     Seek(0, SeekTarget::PrevSyncPoint);
580     return;
581   } else if (mPlayState == PLAY_STATE_LOADING) {
582     mNextState = PLAY_STATE_PLAYING;
583     return;
584   }
585 
586   ChangeState(PLAY_STATE_PLAYING);
587 }
588 
Seek(double aTime,SeekTarget::Type aSeekType)589 void MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) {
590   MOZ_ASSERT(NS_IsMainThread());
591   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
592 
593   MOZ_ASSERT(aTime >= 0.0, "Cannot seek to a negative value.");
594 
595   LOG("Seek");
596   auto time = TimeUnit::FromSeconds(aTime);
597 
598   mLogicalPosition = aTime;
599   mLogicallySeeking = true;
600   SeekTarget target = SeekTarget(time, aSeekType);
601   CallSeek(target);
602 
603   if (mPlayState == PLAY_STATE_ENDED) {
604     ChangeState(GetOwner()->GetPaused() ? PLAY_STATE_PAUSED
605                                         : PLAY_STATE_PLAYING);
606   }
607 }
608 
SetDelaySeekMode(bool aShouldDelaySeek)609 void MediaDecoder::SetDelaySeekMode(bool aShouldDelaySeek) {
610   MOZ_ASSERT(NS_IsMainThread());
611   LOG("SetDelaySeekMode, shouldDelaySeek=%d", aShouldDelaySeek);
612   if (mShouldDelaySeek == aShouldDelaySeek) {
613     return;
614   }
615   mShouldDelaySeek = aShouldDelaySeek;
616   if (!mShouldDelaySeek && mDelayedSeekTarget) {
617     Seek(mDelayedSeekTarget->GetTime().ToSeconds(),
618          mDelayedSeekTarget->GetType());
619     mDelayedSeekTarget.reset();
620   }
621 }
622 
DiscardOngoingSeekIfExists()623 void MediaDecoder::DiscardOngoingSeekIfExists() {
624   MOZ_ASSERT(NS_IsMainThread());
625   mSeekRequest.DisconnectIfExists();
626 }
627 
CallSeek(const SeekTarget & aTarget)628 void MediaDecoder::CallSeek(const SeekTarget& aTarget) {
629   MOZ_ASSERT(NS_IsMainThread());
630   if (mShouldDelaySeek) {
631     LOG("Delay seek to %f and store it to delayed seek target",
632         mDelayedSeekTarget->GetTime().ToSeconds());
633     mDelayedSeekTarget = Some(aTarget);
634     return;
635   }
636   DiscardOngoingSeekIfExists();
637   mDecoderStateMachine->InvokeSeek(aTarget)
638       ->Then(mAbstractMainThread, __func__, this, &MediaDecoder::OnSeekResolved,
639              &MediaDecoder::OnSeekRejected)
640       ->Track(mSeekRequest);
641 }
642 
GetCurrentTime()643 double MediaDecoder::GetCurrentTime() {
644   MOZ_ASSERT(NS_IsMainThread());
645   return mLogicalPosition;
646 }
647 
OnMetadataUpdate(TimedMetadata && aMetadata)648 void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) {
649   MOZ_ASSERT(NS_IsMainThread());
650   MetadataLoaded(MakeUnique<MediaInfo>(*aMetadata.mInfo),
651                  UniquePtr<MetadataTags>(std::move(aMetadata.mTags)),
652                  MediaDecoderEventVisibility::Observable);
653   FirstFrameLoaded(std::move(aMetadata.mInfo),
654                    MediaDecoderEventVisibility::Observable);
655 }
656 
MetadataLoaded(UniquePtr<MediaInfo> aInfo,UniquePtr<MetadataTags> aTags,MediaDecoderEventVisibility aEventVisibility)657 void MediaDecoder::MetadataLoaded(
658     UniquePtr<MediaInfo> aInfo, UniquePtr<MetadataTags> aTags,
659     MediaDecoderEventVisibility aEventVisibility) {
660   MOZ_ASSERT(NS_IsMainThread());
661   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
662 
663   LOG("MetadataLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d",
664       aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(),
665       aInfo->HasVideo());
666 
667   mMediaSeekable = aInfo->mMediaSeekable;
668   mMediaSeekableOnlyInBufferedRanges =
669       aInfo->mMediaSeekableOnlyInBufferedRanges;
670   mInfo = std::move(aInfo);
671 
672   // Make sure the element and the frame (if any) are told about
673   // our new size.
674   if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) {
675     mFiredMetadataLoaded = true;
676     GetOwner()->MetadataLoaded(mInfo.get(), std::move(aTags));
677   }
678   // Invalidate() will end up calling GetOwner()->UpdateMediaSize with the last
679   // dimensions retrieved from the video frame container. The video frame
680   // container contains more up to date dimensions than aInfo.
681   // So we call Invalidate() after calling GetOwner()->MetadataLoaded to ensure
682   // the media element has the latest dimensions.
683   Invalidate();
684 
685   EnsureTelemetryReported();
686 }
687 
EnsureTelemetryReported()688 void MediaDecoder::EnsureTelemetryReported() {
689   MOZ_ASSERT(NS_IsMainThread());
690 
691   if (mTelemetryReported || !mInfo) {
692     // Note: sometimes we get multiple MetadataLoaded calls (for example
693     // for chained ogg). So we ensure we don't report duplicate results for
694     // these resources.
695     return;
696   }
697 
698   nsTArray<nsCString> codecs;
699   if (mInfo->HasAudio() &&
700       !mInfo->mAudio.GetAsAudioInfo()->mMimeType.IsEmpty()) {
701     codecs.AppendElement(mInfo->mAudio.GetAsAudioInfo()->mMimeType);
702   }
703   if (mInfo->HasVideo() &&
704       !mInfo->mVideo.GetAsVideoInfo()->mMimeType.IsEmpty()) {
705     codecs.AppendElement(mInfo->mVideo.GetAsVideoInfo()->mMimeType);
706   }
707   if (codecs.IsEmpty()) {
708     codecs.AppendElement(nsPrintfCString(
709         "resource; %s", ContainerType().OriginalString().Data()));
710   }
711   for (const nsCString& codec : codecs) {
712     LOG("Telemetry MEDIA_CODEC_USED= '%s'", codec.get());
713     Telemetry::Accumulate(Telemetry::HistogramID::MEDIA_CODEC_USED, codec);
714   }
715 
716   mTelemetryReported = true;
717 }
718 
PlayStateStr()719 const char* MediaDecoder::PlayStateStr() {
720   MOZ_ASSERT(NS_IsMainThread());
721   return ToPlayStateStr(mPlayState);
722 }
723 
FirstFrameLoaded(UniquePtr<MediaInfo> aInfo,MediaDecoderEventVisibility aEventVisibility)724 void MediaDecoder::FirstFrameLoaded(
725     UniquePtr<MediaInfo> aInfo, MediaDecoderEventVisibility aEventVisibility) {
726   MOZ_ASSERT(NS_IsMainThread());
727   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
728 
729   LOG("FirstFrameLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d "
730       "mPlayState=%s transportSeekable=%d",
731       aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(),
732       aInfo->HasVideo(), PlayStateStr(), IsTransportSeekable());
733 
734   mInfo = std::move(aInfo);
735 
736   Invalidate();
737 
738   // The element can run javascript via events
739   // before reaching here, so only change the
740   // state if we're still set to the original
741   // loading state.
742   if (mPlayState == PLAY_STATE_LOADING) {
743     ChangeState(mNextState);
744   }
745 
746   // GetOwner()->FirstFrameLoaded() might call us back. Put it at the bottom of
747   // this function to avoid unexpected shutdown from reentrant calls.
748   if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) {
749     GetOwner()->FirstFrameLoaded();
750   }
751 }
752 
NetworkError(const MediaResult & aError)753 void MediaDecoder::NetworkError(const MediaResult& aError) {
754   MOZ_ASSERT(NS_IsMainThread());
755   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
756   GetOwner()->NetworkError(aError);
757 }
758 
DecodeError(const MediaResult & aError)759 void MediaDecoder::DecodeError(const MediaResult& aError) {
760   MOZ_ASSERT(NS_IsMainThread());
761   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
762   GetOwner()->DecodeError(aError);
763 }
764 
UpdateSameOriginStatus(bool aSameOrigin)765 void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) {
766   MOZ_ASSERT(NS_IsMainThread());
767   mSameOriginMedia = aSameOrigin;
768 }
769 
IsSeeking() const770 bool MediaDecoder::IsSeeking() const {
771   MOZ_ASSERT(NS_IsMainThread());
772   return mLogicallySeeking;
773 }
774 
OwnerHasError() const775 bool MediaDecoder::OwnerHasError() const {
776   MOZ_ASSERT(NS_IsMainThread());
777   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
778   return GetOwner()->HasError();
779 }
780 
IsEnded() const781 bool MediaDecoder::IsEnded() const {
782   MOZ_ASSERT(NS_IsMainThread());
783   return mPlayState == PLAY_STATE_ENDED;
784 }
785 
IsShutdown() const786 bool MediaDecoder::IsShutdown() const {
787   MOZ_ASSERT(NS_IsMainThread());
788   return mPlayState == PLAY_STATE_SHUTDOWN;
789 }
790 
PlaybackEnded()791 void MediaDecoder::PlaybackEnded() {
792   MOZ_ASSERT(NS_IsMainThread());
793   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
794 
795   if (mLogicallySeeking || mPlayState == PLAY_STATE_LOADING ||
796       mPlayState == PLAY_STATE_ENDED) {
797     LOG("MediaDecoder::PlaybackEnded bailed out, "
798         "mLogicallySeeking=%d mPlayState=%s",
799         mLogicallySeeking.Ref(), ToPlayStateStr(mPlayState));
800     return;
801   }
802 
803   LOG("MediaDecoder::PlaybackEnded");
804 
805   ChangeState(PLAY_STATE_ENDED);
806   InvalidateWithFlags(VideoFrameContainer::INVALIDATE_FORCE);
807   GetOwner()->PlaybackEnded();
808 }
809 
NotifyPrincipalChanged()810 void MediaDecoder::NotifyPrincipalChanged() {
811   MOZ_ASSERT(NS_IsMainThread());
812   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
813   GetOwner()->NotifyDecoderPrincipalChanged();
814 }
815 
OnSeekResolved()816 void MediaDecoder::OnSeekResolved() {
817   MOZ_ASSERT(NS_IsMainThread());
818   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
819   mLogicallySeeking = false;
820 
821   // Ensure logical position is updated after seek.
822   UpdateLogicalPositionInternal();
823   mSeekRequest.Complete();
824 
825   GetOwner()->SeekCompleted();
826 }
827 
OnSeekRejected()828 void MediaDecoder::OnSeekRejected() {
829   MOZ_ASSERT(NS_IsMainThread());
830   mSeekRequest.Complete();
831   mLogicallySeeking = false;
832 
833   GetOwner()->SeekAborted();
834 }
835 
SeekingStarted()836 void MediaDecoder::SeekingStarted() {
837   MOZ_ASSERT(NS_IsMainThread());
838   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
839   GetOwner()->SeekStarted();
840 }
841 
ChangeState(PlayState aState)842 void MediaDecoder::ChangeState(PlayState aState) {
843   MOZ_ASSERT(NS_IsMainThread());
844   MOZ_ASSERT(!IsShutdown(), "SHUTDOWN is the final state.");
845 
846   if (mNextState == aState) {
847     mNextState = PLAY_STATE_PAUSED;
848   }
849 
850   if (mPlayState != aState) {
851     DDLOG(DDLogCategory::Property, "play_state", ToPlayStateStr(aState));
852   }
853   LOG("Play state changes from %s to %s", ToPlayStateStr(mPlayState),
854       ToPlayStateStr(aState));
855   mPlayState = aState;
856   UpdateTelemetryHelperBasedOnPlayState(aState);
857 }
858 
OwnerVisibility() const859 TelemetryProbesReporter::Visibility MediaDecoder::OwnerVisibility() const {
860   return GetOwner()->IsActuallyInvisible() || mForcedHidden
861              ? TelemetryProbesReporter::Visibility::eInvisible
862              : TelemetryProbesReporter::Visibility::eVisible;
863 }
864 
UpdateTelemetryHelperBasedOnPlayState(PlayState aState) const865 void MediaDecoder::UpdateTelemetryHelperBasedOnPlayState(
866     PlayState aState) const {
867   if (aState == PlayState::PLAY_STATE_PLAYING) {
868     mTelemetryProbesReporter->OnPlay(OwnerVisibility());
869   } else if (aState == PlayState::PLAY_STATE_PAUSED ||
870              aState == PlayState::PLAY_STATE_ENDED) {
871     mTelemetryProbesReporter->OnPause(OwnerVisibility());
872   } else if (aState == PLAY_STATE_SHUTDOWN) {
873     mTelemetryProbesReporter->OnShutdown();
874   }
875 }
876 
GetPositionUpdateReason(double aPrevPos,double aCurPos) const877 MediaDecoder::PositionUpdate MediaDecoder::GetPositionUpdateReason(
878     double aPrevPos, double aCurPos) const {
879   MOZ_ASSERT(NS_IsMainThread());
880   // If current position is earlier than previous position and we didn't do
881   // seek, that means we looped back to the start position, which currently
882   // happens on audio only.
883   const bool notSeeking = !mSeekRequest.Exists();
884   if (mLooping && notSeeking && aCurPos < aPrevPos) {
885     return PositionUpdate::eSeamlessLoopingSeeking;
886   }
887   return aPrevPos != aCurPos && notSeeking ? PositionUpdate::ePeriodicUpdate
888                                            : PositionUpdate::eOther;
889 }
890 
UpdateLogicalPositionInternal()891 void MediaDecoder::UpdateLogicalPositionInternal() {
892   MOZ_ASSERT(NS_IsMainThread());
893   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
894 
895   double currentPosition = CurrentPosition().ToSeconds();
896   if (mPlayState == PLAY_STATE_ENDED) {
897     currentPosition = std::max(currentPosition, mDuration);
898   }
899 
900   const PositionUpdate reason =
901       GetPositionUpdateReason(mLogicalPosition, currentPosition);
902   switch (reason) {
903     case PositionUpdate::ePeriodicUpdate:
904       SetLogicalPosition(currentPosition);
905       // This is actually defined in `TimeMarchesOn`, but we do that in decoder.
906       // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:event-media-timeupdate-7
907       // TODO (bug 1688137): should we move it back to `TimeMarchesOn`?
908       GetOwner()->MaybeQueueTimeupdateEvent();
909       break;
910     case PositionUpdate::eSeamlessLoopingSeeking:
911       // When seamless seeking occurs, seeking was performed on the demuxer so
912       // the decoder doesn't know. That means decoder still thinks it's in
913       // playing. Therefore, we have to manually call those methods to notify
914       // the owner about seeking.
915       GetOwner()->SeekStarted();
916       SetLogicalPosition(currentPosition);
917       GetOwner()->SeekCompleted();
918       break;
919     default:
920       MOZ_ASSERT(reason == PositionUpdate::eOther);
921       SetLogicalPosition(currentPosition);
922       break;
923   }
924 
925   // Invalidate the frame so any video data is displayed.
926   // Do this before the timeupdate event so that if that
927   // event runs JavaScript that queries the media size, the
928   // frame has reflowed and the size updated beforehand.
929   Invalidate();
930 }
931 
SetLogicalPosition(double aNewPosition)932 void MediaDecoder::SetLogicalPosition(double aNewPosition) {
933   MOZ_ASSERT(NS_IsMainThread());
934   if (mLogicalPosition == aNewPosition) {
935     return;
936   }
937   mLogicalPosition = aNewPosition;
938   DDLOG(DDLogCategory::Property, "currentTime", mLogicalPosition);
939 }
940 
DurationChanged()941 void MediaDecoder::DurationChanged() {
942   MOZ_ASSERT(NS_IsMainThread());
943   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
944 
945   double oldDuration = mDuration;
946 
947   // Use the explicit duration if we have one.
948   // Otherwise use the duration mirrored from MDSM.
949   if (mExplicitDuration.isSome()) {
950     mDuration = mExplicitDuration.ref();
951   } else if (mStateMachineDuration.Ref().isSome()) {
952     mDuration = mStateMachineDuration.Ref().ref().ToSeconds();
953   }
954 
955   if (mDuration == oldDuration || IsNaN(mDuration)) {
956     return;
957   }
958 
959   LOG("Duration changed to %f", mDuration);
960 
961   // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=28822 for a discussion
962   // of whether we should fire durationchange on explicit infinity.
963   if (mFiredMetadataLoaded &&
964       (!mozilla::IsInfinite<double>(mDuration) || mExplicitDuration.isSome())) {
965     GetOwner()->DispatchAsyncEvent(u"durationchange"_ns);
966   }
967 
968   if (CurrentPosition() > TimeUnit::FromSeconds(mDuration)) {
969     Seek(mDuration, SeekTarget::Accurate);
970   }
971 }
972 
GetCompositor()973 already_AddRefed<KnowsCompositor> MediaDecoder::GetCompositor() {
974   MediaDecoderOwner* owner = GetOwner();
975   Document* ownerDoc = owner ? owner->GetDocument() : nullptr;
976   RefPtr<LayerManager> layerManager =
977       ownerDoc ? nsContentUtils::LayerManagerForDocument(ownerDoc) : nullptr;
978   RefPtr<KnowsCompositor> knows =
979       layerManager ? layerManager->AsKnowsCompositor() : nullptr;
980   return knows ? knows->GetForMedia().forget() : nullptr;
981 }
982 
NotifyCompositor()983 void MediaDecoder::NotifyCompositor() {
984   RefPtr<KnowsCompositor> knowsCompositor = GetCompositor();
985   if (knowsCompositor) {
986     nsCOMPtr<nsIRunnable> r =
987         NewRunnableMethod<already_AddRefed<KnowsCompositor>&&>(
988             "MediaFormatReader::UpdateCompositor", mReader,
989             &MediaFormatReader::UpdateCompositor, knowsCompositor.forget());
990     Unused << mReader->OwnerThread()->Dispatch(r.forget());
991   }
992 }
993 
SetElementVisibility(bool aIsOwnerInvisible,bool aIsOwnerConnected)994 void MediaDecoder::SetElementVisibility(bool aIsOwnerInvisible,
995                                         bool aIsOwnerConnected) {
996   MOZ_ASSERT(NS_IsMainThread());
997   mIsOwnerInvisible = aIsOwnerInvisible;
998   mIsOwnerConnected = aIsOwnerConnected;
999   mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility());
1000   UpdateVideoDecodeMode();
1001 }
1002 
SetForcedHidden(bool aForcedHidden)1003 void MediaDecoder::SetForcedHidden(bool aForcedHidden) {
1004   MOZ_ASSERT(NS_IsMainThread());
1005   mForcedHidden = aForcedHidden;
1006   mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility());
1007   UpdateVideoDecodeMode();
1008 }
1009 
SetSuspendTaint(bool aTainted)1010 void MediaDecoder::SetSuspendTaint(bool aTainted) {
1011   MOZ_ASSERT(NS_IsMainThread());
1012   mHasSuspendTaint = aTainted;
1013   UpdateVideoDecodeMode();
1014 }
1015 
UpdateVideoDecodeMode()1016 void MediaDecoder::UpdateVideoDecodeMode() {
1017   MOZ_ASSERT(NS_IsMainThread());
1018 
1019   // The MDSM may yet be set.
1020   if (!mDecoderStateMachine) {
1021     LOG("UpdateVideoDecodeMode(), early return because we don't have MDSM.");
1022     return;
1023   }
1024 
1025   // Seeking is required when leaving suspend mode.
1026   if (!mMediaSeekable) {
1027     LOG("UpdateVideoDecodeMode(), set Normal because the media is not "
1028         "seekable");
1029     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1030     return;
1031   }
1032 
1033   // If mHasSuspendTaint is set, never suspend the video decoder.
1034   if (mHasSuspendTaint) {
1035     LOG("UpdateVideoDecodeMode(), set Normal because the element has been "
1036         "tainted.");
1037     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1038     return;
1039   }
1040 
1041   // If mSecondaryVideoContainer is set, never suspend the video decoder.
1042   if (mSecondaryVideoContainer.Ref()) {
1043     LOG("UpdateVideoDecodeMode(), set Normal because the element is cloning "
1044         "itself visually to another video container.");
1045     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1046     return;
1047   }
1048 
1049   // Don't suspend elements that is not in a connected tree.
1050   if (!mIsOwnerConnected) {
1051     LOG("UpdateVideoDecodeMode(), set Normal because the element is not in "
1052         "tree.");
1053     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1054     return;
1055   }
1056 
1057   // If mForcedHidden is set, suspend the video decoder anyway.
1058   if (mForcedHidden) {
1059     LOG("UpdateVideoDecodeMode(), set Suspend because the element is forced to "
1060         "be suspended.");
1061     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend);
1062     return;
1063   }
1064 
1065   // Resume decoding in the advance, even the element is in the background.
1066   if (mIsBackgroundVideoDecodingAllowed) {
1067     LOG("UpdateVideoDecodeMode(), set Normal because the tab is in background "
1068         "and hovered.");
1069     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1070     return;
1071   }
1072 
1073   if (mIsOwnerInvisible) {
1074     LOG("UpdateVideoDecodeMode(), set Suspend because of invisible element.");
1075     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend);
1076   } else {
1077     LOG("UpdateVideoDecodeMode(), set Normal because of visible element.");
1078     mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
1079   }
1080 }
1081 
SetIsBackgroundVideoDecodingAllowed(bool aAllowed)1082 void MediaDecoder::SetIsBackgroundVideoDecodingAllowed(bool aAllowed) {
1083   mIsBackgroundVideoDecodingAllowed = aAllowed;
1084   UpdateVideoDecodeMode();
1085 }
1086 
HasSuspendTaint() const1087 bool MediaDecoder::HasSuspendTaint() const {
1088   MOZ_ASSERT(NS_IsMainThread());
1089   return mHasSuspendTaint;
1090 }
1091 
SetSecondaryVideoContainer(RefPtr<VideoFrameContainer> aSecondaryVideoContainer)1092 void MediaDecoder::SetSecondaryVideoContainer(
1093     RefPtr<VideoFrameContainer> aSecondaryVideoContainer) {
1094   MOZ_ASSERT(NS_IsMainThread());
1095   if (mSecondaryVideoContainer.Ref() == aSecondaryVideoContainer) {
1096     return;
1097   }
1098   mSecondaryVideoContainer = std::move(aSecondaryVideoContainer);
1099   UpdateVideoDecodeMode();
1100 }
1101 
IsMediaSeekable()1102 bool MediaDecoder::IsMediaSeekable() {
1103   MOZ_ASSERT(NS_IsMainThread());
1104   NS_ENSURE_TRUE(GetStateMachine(), false);
1105   return mMediaSeekable;
1106 }
1107 
GetSeekable()1108 media::TimeIntervals MediaDecoder::GetSeekable() {
1109   MOZ_ASSERT(NS_IsMainThread());
1110 
1111   if (IsNaN(GetDuration())) {
1112     // We do not have a duration yet, we can't determine the seekable range.
1113     return TimeIntervals();
1114   }
1115 
1116   // We can seek in buffered range if the media is seekable. Also, we can seek
1117   // in unbuffered ranges if the transport level is seekable (local file or the
1118   // server supports range requests, etc.) or in cue-less WebMs
1119   if (mMediaSeekableOnlyInBufferedRanges) {
1120     return GetBuffered();
1121   } else if (!IsMediaSeekable()) {
1122     return media::TimeIntervals();
1123   } else if (!IsTransportSeekable()) {
1124     return GetBuffered();
1125   } else {
1126     return media::TimeIntervals(media::TimeInterval(
1127         TimeUnit::Zero(), IsInfinite() ? TimeUnit::FromInfinity()
1128                                        : TimeUnit::FromSeconds(GetDuration())));
1129   }
1130 }
1131 
SetFragmentEndTime(double aTime)1132 void MediaDecoder::SetFragmentEndTime(double aTime) {
1133   MOZ_ASSERT(NS_IsMainThread());
1134   if (mDecoderStateMachine) {
1135     mDecoderStateMachine->DispatchSetFragmentEndTime(
1136         TimeUnit::FromSeconds(aTime));
1137   }
1138 }
1139 
SetPlaybackRate(double aPlaybackRate)1140 void MediaDecoder::SetPlaybackRate(double aPlaybackRate) {
1141   MOZ_ASSERT(NS_IsMainThread());
1142 
1143   double oldRate = mPlaybackRate;
1144   mPlaybackRate = aPlaybackRate;
1145   if (aPlaybackRate == 0) {
1146     Pause();
1147     return;
1148   }
1149 
1150   if (oldRate == 0 && !GetOwner()->GetPaused()) {
1151     // PlaybackRate is no longer null.
1152     // Restart the playback if the media was playing.
1153     Play();
1154   }
1155 
1156   if (mDecoderStateMachine) {
1157     mDecoderStateMachine->DispatchSetPlaybackRate(aPlaybackRate);
1158   }
1159 }
1160 
SetPreservesPitch(bool aPreservesPitch)1161 void MediaDecoder::SetPreservesPitch(bool aPreservesPitch) {
1162   MOZ_ASSERT(NS_IsMainThread());
1163   mPreservesPitch = aPreservesPitch;
1164 }
1165 
SetLooping(bool aLooping)1166 void MediaDecoder::SetLooping(bool aLooping) {
1167   MOZ_ASSERT(NS_IsMainThread());
1168   mLooping = aLooping;
1169 }
1170 
SetStreamName(const nsAutoString & aStreamName)1171 void MediaDecoder::SetStreamName(const nsAutoString& aStreamName) {
1172   MOZ_ASSERT(NS_IsMainThread());
1173   mStreamName = aStreamName;
1174 }
1175 
ConnectMirrors(MediaDecoderStateMachine * aObject)1176 void MediaDecoder::ConnectMirrors(MediaDecoderStateMachine* aObject) {
1177   MOZ_ASSERT(NS_IsMainThread());
1178   MOZ_ASSERT(aObject);
1179   mStateMachineDuration.Connect(aObject->CanonicalDuration());
1180   mBuffered.Connect(aObject->CanonicalBuffered());
1181   mCurrentPosition.Connect(aObject->CanonicalCurrentPosition());
1182   mIsAudioDataAudible.Connect(aObject->CanonicalIsAudioDataAudible());
1183 }
1184 
DisconnectMirrors()1185 void MediaDecoder::DisconnectMirrors() {
1186   MOZ_ASSERT(NS_IsMainThread());
1187   mStateMachineDuration.DisconnectIfConnected();
1188   mBuffered.DisconnectIfConnected();
1189   mCurrentPosition.DisconnectIfConnected();
1190   mIsAudioDataAudible.DisconnectIfConnected();
1191 }
1192 
SetStateMachine(MediaDecoderStateMachine * aStateMachine)1193 void MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine) {
1194   MOZ_ASSERT(NS_IsMainThread());
1195   MOZ_ASSERT_IF(aStateMachine, !mDecoderStateMachine);
1196   if (aStateMachine) {
1197     mDecoderStateMachine = aStateMachine;
1198     DDLINKCHILD("decoder state machine", mDecoderStateMachine.get());
1199     ConnectMirrors(aStateMachine);
1200     UpdateVideoDecodeMode();
1201   } else if (mDecoderStateMachine) {
1202     DDUNLINKCHILD(mDecoderStateMachine.get());
1203     mDecoderStateMachine = nullptr;
1204     DisconnectMirrors();
1205   }
1206 }
1207 
GetImageContainer()1208 ImageContainer* MediaDecoder::GetImageContainer() {
1209   return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer()
1210                               : nullptr;
1211 }
1212 
InvalidateWithFlags(uint32_t aFlags)1213 void MediaDecoder::InvalidateWithFlags(uint32_t aFlags) {
1214   if (mVideoFrameContainer) {
1215     mVideoFrameContainer->InvalidateWithFlags(aFlags);
1216   }
1217 }
1218 
Invalidate()1219 void MediaDecoder::Invalidate() {
1220   if (mVideoFrameContainer) {
1221     mVideoFrameContainer->Invalidate();
1222   }
1223 }
1224 
Suspend()1225 void MediaDecoder::Suspend() {
1226   MOZ_ASSERT(NS_IsMainThread());
1227   GetStateMachine()->InvokeSuspendMediaSink();
1228 }
1229 
Resume()1230 void MediaDecoder::Resume() {
1231   MOZ_ASSERT(NS_IsMainThread());
1232   GetStateMachine()->InvokeResumeMediaSink();
1233 }
1234 
1235 // Constructs the time ranges representing what segments of the media
1236 // are buffered and playable.
GetBuffered()1237 media::TimeIntervals MediaDecoder::GetBuffered() {
1238   MOZ_ASSERT(NS_IsMainThread());
1239   return mBuffered.Ref();
1240 }
1241 
SizeOfVideoQueue()1242 size_t MediaDecoder::SizeOfVideoQueue() {
1243   MOZ_ASSERT(NS_IsMainThread());
1244   if (mDecoderStateMachine) {
1245     return mDecoderStateMachine->SizeOfVideoQueue();
1246   }
1247   return 0;
1248 }
1249 
SizeOfAudioQueue()1250 size_t MediaDecoder::SizeOfAudioQueue() {
1251   MOZ_ASSERT(NS_IsMainThread());
1252   if (mDecoderStateMachine) {
1253     return mDecoderStateMachine->SizeOfAudioQueue();
1254   }
1255   return 0;
1256 }
1257 
NotifyReaderDataArrived()1258 void MediaDecoder::NotifyReaderDataArrived() {
1259   MOZ_ASSERT(NS_IsMainThread());
1260   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1261 
1262   nsresult rv = mReader->OwnerThread()->Dispatch(
1263       NewRunnableMethod("MediaFormatReader::NotifyDataArrived", mReader.get(),
1264                         &MediaFormatReader::NotifyDataArrived));
1265   MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
1266   Unused << rv;
1267 }
1268 
1269 // Provide access to the state machine object
GetStateMachine() const1270 MediaDecoderStateMachine* MediaDecoder::GetStateMachine() const {
1271   MOZ_ASSERT(NS_IsMainThread());
1272   return mDecoderStateMachine;
1273 }
1274 
CanPlayThrough()1275 bool MediaDecoder::CanPlayThrough() {
1276   MOZ_ASSERT(NS_IsMainThread());
1277   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1278   return CanPlayThroughImpl();
1279 }
1280 
SetCDMProxy(CDMProxy * aProxy)1281 RefPtr<SetCDMPromise> MediaDecoder::SetCDMProxy(CDMProxy* aProxy) {
1282   MOZ_ASSERT(NS_IsMainThread());
1283   return InvokeAsync<RefPtr<CDMProxy>>(mReader->OwnerThread(), mReader.get(),
1284                                        __func__,
1285                                        &MediaFormatReader::SetCDMProxy, aProxy);
1286 }
1287 
IsOpusEnabled()1288 bool MediaDecoder::IsOpusEnabled() { return StaticPrefs::media_opus_enabled(); }
1289 
IsOggEnabled()1290 bool MediaDecoder::IsOggEnabled() { return StaticPrefs::media_ogg_enabled(); }
1291 
IsWaveEnabled()1292 bool MediaDecoder::IsWaveEnabled() { return StaticPrefs::media_wave_enabled(); }
1293 
IsWebMEnabled()1294 bool MediaDecoder::IsWebMEnabled() { return StaticPrefs::media_webm_enabled(); }
1295 
1296 NS_IMETHODIMP
CollectReports(nsIHandleReportCallback * aHandleReport,nsISupports * aData,bool aAnonymize)1297 MediaMemoryTracker::CollectReports(nsIHandleReportCallback* aHandleReport,
1298                                    nsISupports* aData, bool aAnonymize) {
1299   // NB: When resourceSizes' ref count goes to 0 the promise will report the
1300   //     resources memory and finish the asynchronous memory report.
1301   RefPtr<MediaDecoder::ResourceSizes> resourceSizes =
1302       new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf);
1303 
1304   nsCOMPtr<nsIHandleReportCallback> handleReport = aHandleReport;
1305   nsCOMPtr<nsISupports> data = aData;
1306 
1307   resourceSizes->Promise()->Then(
1308       AbstractThread::MainThread(), __func__,
1309       [handleReport, data](size_t size) {
1310         handleReport->Callback(
1311             ""_ns, "explicit/media/resources"_ns, KIND_HEAP, UNITS_BYTES, size,
1312             nsLiteralCString("Memory used by media resources including "
1313                              "streaming buffers, caches, etc."),
1314             data);
1315 
1316         nsCOMPtr<nsIMemoryReporterManager> imgr =
1317             do_GetService("@mozilla.org/memory-reporter-manager;1");
1318 
1319         if (imgr) {
1320           imgr->EndReport();
1321         }
1322       },
1323       [](size_t) { /* unused reject function */ });
1324 
1325   int64_t video = 0;
1326   int64_t audio = 0;
1327   DecodersArray& decoders = Decoders();
1328   for (size_t i = 0; i < decoders.Length(); ++i) {
1329     MediaDecoder* decoder = decoders[i];
1330     video += decoder->SizeOfVideoQueue();
1331     audio += decoder->SizeOfAudioQueue();
1332     decoder->AddSizeOfResources(resourceSizes);
1333   }
1334 
1335   MOZ_COLLECT_REPORT("explicit/media/decoded/video", KIND_HEAP, UNITS_BYTES,
1336                      video, "Memory used by decoded video frames.");
1337 
1338   MOZ_COLLECT_REPORT("explicit/media/decoded/audio", KIND_HEAP, UNITS_BYTES,
1339                      audio, "Memory used by decoded audio chunks.");
1340 
1341   return NS_OK;
1342 }
1343 
GetOwner() const1344 MediaDecoderOwner* MediaDecoder::GetOwner() const {
1345   MOZ_ASSERT(NS_IsMainThread());
1346   // mOwner is valid until shutdown.
1347   return mOwner;
1348 }
1349 
NextFrameBufferedStatus()1350 MediaDecoderOwner::NextFrameStatus MediaDecoder::NextFrameBufferedStatus() {
1351   MOZ_ASSERT(NS_IsMainThread());
1352   // Next frame hasn't been decoded yet.
1353   // Use the buffered range to consider if we have the next frame available.
1354   auto currentPosition = CurrentPosition();
1355   media::TimeInterval interval(
1356       currentPosition, currentPosition + DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED);
1357   return GetBuffered().Contains(interval)
1358              ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
1359              : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
1360 }
1361 
GetDebugInfo(dom::MediaDecoderDebugInfo & aInfo)1362 void MediaDecoder::GetDebugInfo(dom::MediaDecoderDebugInfo& aInfo) {
1363   CopyUTF8toUTF16(nsPrintfCString("%p", this), aInfo.mInstance);
1364   aInfo.mChannels = mInfo ? mInfo->mAudio.mChannels : 0;
1365   aInfo.mRate = mInfo ? mInfo->mAudio.mRate : 0;
1366   aInfo.mHasAudio = mInfo ? mInfo->HasAudio() : false;
1367   aInfo.mHasVideo = mInfo ? mInfo->HasVideo() : false;
1368   CopyUTF8toUTF16(MakeStringSpan(PlayStateStr()), aInfo.mPlayState);
1369   aInfo.mContainerType =
1370       NS_ConvertUTF8toUTF16(ContainerType().Type().AsString());
1371   mReader->GetDebugInfo(aInfo.mReader);
1372 }
1373 
RequestDebugInfo(MediaDecoderDebugInfo & aInfo)1374 RefPtr<GenericPromise> MediaDecoder::RequestDebugInfo(
1375     MediaDecoderDebugInfo& aInfo) {
1376   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1377   GetDebugInfo(aInfo);
1378 
1379   if (!GetStateMachine()) {
1380     return GenericPromise::CreateAndResolve(true, __func__);
1381   }
1382 
1383   return GetStateMachine()
1384       ->RequestDebugInfo(aInfo.mStateMachine)
1385       ->Then(
1386           AbstractThread::MainThread(), __func__,
1387           []() { return GenericPromise::CreateAndResolve(true, __func__); },
1388           []() {
1389             MOZ_ASSERT_UNREACHABLE("Unexpected RequestDebugInfo() rejection");
1390             return GenericPromise::CreateAndResolve(false, __func__);
1391           });
1392 }
1393 
NotifyAudibleStateChanged()1394 void MediaDecoder::NotifyAudibleStateChanged() {
1395   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
1396   GetOwner()->SetAudibleState(mIsAudioDataAudible);
1397 }
1398 
GetTotalPlayTimeInSeconds() const1399 double MediaDecoder::GetTotalPlayTimeInSeconds() const {
1400   return mTelemetryProbesReporter->GetTotalPlayTimeInSeconds();
1401 }
1402 
GetInvisibleVideoPlayTimeInSeconds() const1403 double MediaDecoder::GetInvisibleVideoPlayTimeInSeconds() const {
1404   return mTelemetryProbesReporter->GetInvisibleVideoPlayTimeInSeconds();
1405 }
1406 
GetVideoDecodeSuspendedTimeInSeconds() const1407 double MediaDecoder::GetVideoDecodeSuspendedTimeInSeconds() const {
1408   return mTelemetryProbesReporter->GetVideoDecodeSuspendedTimeInSeconds();
1409 }
1410 
1411 MediaMemoryTracker::MediaMemoryTracker() = default;
1412 
InitMemoryReporter()1413 void MediaMemoryTracker::InitMemoryReporter() {
1414   RegisterWeakAsyncMemoryReporter(this);
1415 }
1416 
~MediaMemoryTracker()1417 MediaMemoryTracker::~MediaMemoryTracker() {
1418   UnregisterWeakMemoryReporter(this);
1419 }
1420 
1421 }  // namespace mozilla
1422 
1423 // avoid redefined macro in unified build
1424 #undef DUMP
1425 #undef LOG
1426 #undef NS_DispatchToMainThread
1427