1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #if !defined(MediaDecoder_h_) 8 # define MediaDecoder_h_ 9 10 # include "BackgroundVideoDecodingPermissionObserver.h" 11 # include "DecoderDoctorDiagnostics.h" 12 # include "MediaContainerType.h" 13 # include "MediaDecoderOwner.h" 14 # include "MediaEventSource.h" 15 # include "MediaMetadataManager.h" 16 # include "MediaPromiseDefs.h" 17 # include "MediaResource.h" 18 # include "MediaStatistics.h" 19 # include "SeekTarget.h" 20 # include "TelemetryProbesReporter.h" 21 # include "TimeUnits.h" 22 # include "mozilla/Atomics.h" 23 # include "mozilla/CDMProxy.h" 24 # include "mozilla/MozPromise.h" 25 # include "mozilla/ReentrantMonitor.h" 26 # include "mozilla/StateMirroring.h" 27 # include "mozilla/StateWatching.h" 28 # include "mozilla/dom/MediaDebugInfoBinding.h" 29 # include "nsCOMPtr.h" 30 # include "nsIObserver.h" 31 # include "nsISupports.h" 32 # include "nsITimer.h" 33 34 class AudioDeviceInfo; 35 class nsIPrincipal; 36 37 namespace mozilla { 38 39 namespace dom { 40 class MediaMemoryInfo; 41 } 42 43 class AbstractThread; 44 class DOMMediaStream; 45 class DecoderBenchmark; 46 class ProcessedMediaTrack; 47 class FrameStatistics; 48 class VideoFrameContainer; 49 class MediaFormatReader; 50 class MediaDecoderStateMachine; 51 struct MediaPlaybackEvent; 52 struct SharedDummyTrack; 53 54 struct MOZ_STACK_CLASS MediaDecoderInit { 55 MediaDecoderOwner* const mOwner; 56 TelemetryProbesReporterOwner* const mReporterOwner; 57 const double mVolume; 58 const bool mPreservesPitch; 59 const double mPlaybackRate; 60 const bool mMinimizePreroll; 61 const bool mHasSuspendTaint; 62 const bool mLooping; 63 const MediaContainerType mContainerType; 64 const nsAutoString mStreamName; 65 MediaDecoderInitMediaDecoderInit66 MediaDecoderInit(MediaDecoderOwner* aOwner, 67 TelemetryProbesReporterOwner* aReporterOwner, double aVolume, 68 bool aPreservesPitch, double aPlaybackRate, 69 bool aMinimizePreroll, bool aHasSuspendTaint, bool aLooping, 70 const MediaContainerType& aContainerType) 71 : mOwner(aOwner), 72 mReporterOwner(aReporterOwner), 73 mVolume(aVolume), 74 mPreservesPitch(aPreservesPitch), 75 mPlaybackRate(aPlaybackRate), 76 mMinimizePreroll(aMinimizePreroll), 77 mHasSuspendTaint(aHasSuspendTaint), 78 mLooping(aLooping), 79 mContainerType(aContainerType) {} 80 }; 81 82 DDLoggedTypeDeclName(MediaDecoder); 83 84 class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> { 85 public: 86 typedef MozPromise<bool /* aIgnored */, bool /* aIgnored */, 87 /* IsExclusive = */ true> 88 SeekPromise; 89 90 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoder) 91 92 // Enumeration for the valid play states (see mPlayState) 93 enum PlayState { 94 PLAY_STATE_LOADING, 95 PLAY_STATE_PAUSED, 96 PLAY_STATE_PLAYING, 97 PLAY_STATE_ENDED, 98 PLAY_STATE_SHUTDOWN 99 }; 100 101 // Must be called exactly once, on the main thread, during startup. 102 static void InitStatics(); 103 104 explicit MediaDecoder(MediaDecoderInit& aInit); 105 106 // Returns the container content type of the resource. 107 // Safe to call from any thread. ContainerType()108 const MediaContainerType& ContainerType() const { return mContainerType; } 109 110 // Cleanup internal data structures. Must be called on the main 111 // thread by the owning object before that object disposes of this object. 112 virtual void Shutdown(); 113 114 // Notified by the shutdown manager that XPCOM shutdown has begun. 115 // The decoder should notify its owner to drop the reference to the decoder 116 // to prevent further calls into the decoder. 117 void NotifyXPCOMShutdown(); 118 119 // Called if the media file encounters a network error. 120 void NetworkError(const MediaResult& aError); 121 122 // Return the principal of the current URI being played or downloaded. 123 virtual already_AddRefed<nsIPrincipal> GetCurrentPrincipal() = 0; 124 125 // Return true if the loading of this resource required cross-origin 126 // redirects. 127 virtual bool HadCrossOriginRedirects() = 0; 128 129 // Return the time position in the video stream being 130 // played measured in seconds. 131 virtual double GetCurrentTime(); 132 133 // Seek to the time position in (seconds) from the start of the video. 134 // If aDoFastSeek is true, we'll seek to the sync point/keyframe preceeding 135 // the seek target. 136 void Seek(double aTime, SeekTarget::Type aSeekType); 137 138 // Initialize state machine and schedule it. 139 nsresult InitializeStateMachine(); 140 141 // Start playback of a video. 'Load' must have previously been 142 // called. 143 virtual void Play(); 144 145 // Notify activity of the decoder owner is changed. 146 virtual void NotifyOwnerActivityChanged(bool aIsOwnerInvisible, 147 bool aIsOwnerConnected); 148 149 // Pause video playback. 150 virtual void Pause(); 151 // Adjust the speed of the playback, optionally with pitch correction, 152 void SetVolume(double aVolume); 153 154 void SetPlaybackRate(double aPlaybackRate); 155 void SetPreservesPitch(bool aPreservesPitch); 156 void SetLooping(bool aLooping); 157 void SetStreamName(const nsAutoString& aStreamName); 158 159 // Set the given device as the output device. 160 RefPtr<GenericPromise> SetSink(AudioDeviceInfo* aSinkDevice); 161 GetMinimizePreroll()162 bool GetMinimizePreroll() const { return mMinimizePreroll; } 163 164 // When we enable delay seek mode, media decoder won't actually ask MDSM to do 165 // seeking. During this period, we would store the latest seeking target and 166 // perform the seek to that target when we leave the mode. If we have any 167 // delayed seeks stored `IsSeeking()` will return true. E.g. During delay 168 // seeking mode, if we get seek target to 5s, 10s, 7s. When we stop delaying 169 // seeking, we would only seek to 7s. 170 void SetDelaySeekMode(bool aShouldDelaySeek); 171 172 // All MediaStream-related data is protected by mReentrantMonitor. 173 // We have at most one DecodedStreamData per MediaDecoder. Its stream 174 // is used as the input for each ProcessedMediaTrack created by calls to 175 // captureStream(UntilEnded). Seeking creates a new source stream, as does 176 // replaying after the input as ended. In the latter case, the new source is 177 // not connected to streams created by captureStreamUntilEnded. 178 179 enum class OutputCaptureState { Capture, Halt, None }; 180 // Set the output capture state of this decoder. 181 // @param aState Capture: Output is captured into output tracks, and 182 // aDummyTrack must be provided. 183 // Halt: A capturing media sink is used, but capture is 184 // halted. 185 // None: Output is not captured. 186 // @param aDummyTrack A SharedDummyTrack the capturing media sink can use to 187 // access a MediaTrackGraph, so it can create tracks even 188 // when there are no output tracks available. 189 void SetOutputCaptureState(OutputCaptureState aState, 190 SharedDummyTrack* aDummyTrack = nullptr); 191 // Add an output track. All decoder output for the track's media type will be 192 // sent to the track. 193 // Note that only one audio track and one video track is supported by 194 // MediaDecoder at this time. Passing in more of one type, or passing in a 195 // type that metadata says we are not decoding, is an error. 196 void AddOutputTrack(RefPtr<ProcessedMediaTrack> aTrack); 197 // Remove an output track added with AddOutputTrack. 198 void RemoveOutputTrack(const RefPtr<ProcessedMediaTrack>& aTrack); 199 // Update the principal for any output tracks. 200 void SetOutputTracksPrincipal(const RefPtr<nsIPrincipal>& aPrincipal); 201 202 // Return the duration of the video in seconds. 203 virtual double GetDuration(); 204 205 // Return true if the stream is infinite. 206 bool IsInfinite() const; 207 208 // Return true if we are currently seeking in the media resource. 209 // Call on the main thread only. 210 bool IsSeeking() const; 211 212 // Return true if the decoder has reached the end of playback. 213 bool IsEnded() const; 214 215 // True if we are playing a MediaSource object. IsMSE()216 virtual bool IsMSE() const { return false; } 217 218 // Return true if the MediaDecoderOwner's error attribute is not null. 219 // Must be called before Shutdown(). 220 bool OwnerHasError() const; 221 222 // Returns true if this media supports random seeking. False for example with 223 // chained ogg files. 224 bool IsMediaSeekable(); 225 // Returns true if seeking is supported on a transport level (e.g. the server 226 // supports range requests, we are playing a file, etc.). 227 virtual bool IsTransportSeekable() = 0; 228 229 // Return the time ranges that can be seeked into. 230 virtual media::TimeIntervals GetSeekable(); 231 232 // Set the end time of the media resource. When playback reaches 233 // this point the media pauses. aTime is in seconds. 234 virtual void SetFragmentEndTime(double aTime); 235 236 // Invalidate the frame. 237 void Invalidate(); 238 void InvalidateWithFlags(uint32_t aFlags); 239 240 // Suspend any media downloads that are in progress. Called by the 241 // media element when it is sent to the bfcache, or when we need 242 // to throttle the download. Call on the main thread only. This can 243 // be called multiple times, there's an internal "suspend count". 244 // When it is called the internal system audio resource are cleaned up. 245 virtual void Suspend(); 246 247 // Resume any media downloads that have been suspended. Called by the 248 // media element when it is restored from the bfcache, or when we need 249 // to stop throttling the download. Call on the main thread only. 250 // The download will only actually resume once as many Resume calls 251 // have been made as Suspend calls. 252 virtual void Resume(); 253 254 // Moves any existing channel loads into or out of background. Background 255 // loads don't block the load event. This is called when we stop or restart 256 // delaying the load event. This also determines whether any new loads 257 // initiated (for example to seek) will be in the background. This calls 258 // SetLoadInBackground() on mResource. SetLoadInBackground(bool aLoadInBackground)259 virtual void SetLoadInBackground(bool aLoadInBackground) {} 260 261 MediaDecoderStateMachine* GetStateMachine() const; 262 void SetStateMachine(MediaDecoderStateMachine* aStateMachine); 263 264 // Constructs the time ranges representing what segments of the media 265 // are buffered and playable. 266 virtual media::TimeIntervals GetBuffered(); 267 268 // Returns the size, in bytes, of the heap memory used by the currently 269 // queued decoded video and audio data. 270 size_t SizeOfVideoQueue(); 271 size_t SizeOfAudioQueue(); 272 273 // Helper struct for accumulating resource sizes that need to be measured 274 // asynchronously. Once all references are dropped the callback will be 275 // invoked. 276 struct ResourceSizes { 277 typedef MozPromise<size_t, size_t, true> SizeOfPromise; NS_INLINE_DECL_THREADSAFE_REFCOUNTINGResourceSizes278 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ResourceSizes) 279 explicit ResourceSizes(MallocSizeOf aMallocSizeOf) 280 : mMallocSizeOf(aMallocSizeOf), mByteSize(0), mCallback() {} 281 282 mozilla::MallocSizeOf mMallocSizeOf; 283 mozilla::Atomic<size_t> mByteSize; 284 PromiseResourceSizes285 RefPtr<SizeOfPromise> Promise() { return mCallback.Ensure(__func__); } 286 287 private: ~ResourceSizesResourceSizes288 ~ResourceSizes() { mCallback.ResolveIfExists(mByteSize, __func__); } 289 290 MozPromiseHolder<SizeOfPromise> mCallback; 291 }; 292 293 virtual void AddSizeOfResources(ResourceSizes* aSizes) = 0; 294 GetVideoFrameContainer()295 VideoFrameContainer* GetVideoFrameContainer() { return mVideoFrameContainer; } 296 297 layers::ImageContainer* GetImageContainer(); 298 299 // Returns true if we can play the entire media through without stopping 300 // to buffer, given the current download and playback rates. 301 bool CanPlayThrough(); 302 303 // Called from HTMLMediaElement when owner document activity changes 304 virtual void SetElementVisibility(bool aIsOwnerInvisible, 305 bool aIsOwnerConnected); 306 307 // Force override the visible state to hidden. 308 // Called from HTMLMediaElement when testing of video decode suspend from 309 // mochitests. 310 void SetForcedHidden(bool aForcedHidden); 311 312 // Mark the decoder as tainted, meaning suspend-video-decoder is disabled. 313 void SetSuspendTaint(bool aTaint); 314 315 // Returns true if the decoder can't participate in suspend-video-decoder. 316 bool HasSuspendTaint() const; 317 318 void UpdateVideoDecodeMode(); 319 320 void SetSecondaryVideoContainer( 321 const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer); 322 323 void SetIsBackgroundVideoDecodingAllowed(bool aAllowed); 324 325 bool IsVideoDecodingSuspended() const; 326 327 /****** 328 * The following methods must only be called on the main 329 * thread. 330 ******/ 331 332 // Change to a new play state. This updates the mState variable and 333 // notifies any thread blocking on this object's monitor of the 334 // change. Call on the main thread only. 335 virtual void ChangeState(PlayState aState); 336 337 // Called when the video has completed playing. 338 // Call on the main thread only. 339 void PlaybackEnded(); 340 341 void OnSeekRejected(); 342 void OnSeekResolved(); 343 344 // Seeking has started. Inform the element on the main thread. 345 void SeekingStarted(); 346 347 void UpdateLogicalPositionInternal(); UpdateLogicalPosition()348 void UpdateLogicalPosition() { 349 MOZ_ASSERT(NS_IsMainThread()); 350 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); 351 // Per spec, offical position remains stable during pause and seek. 352 if (mPlayState == PLAY_STATE_PAUSED || IsSeeking()) { 353 return; 354 } 355 UpdateLogicalPositionInternal(); 356 } 357 358 // Find the end of the cached data starting at the current decoder 359 // position. 360 int64_t GetDownloadPosition(); 361 362 // Notifies the element that decoding has failed. 363 void DecodeError(const MediaResult& aError); 364 365 // Indicate whether the media is same-origin with the element. 366 void UpdateSameOriginStatus(bool aSameOrigin); 367 368 MediaDecoderOwner* GetOwner() const; 369 AbstractMainThread()370 AbstractThread* AbstractMainThread() const { return mAbstractMainThread; } 371 372 RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy); 373 374 void EnsureTelemetryReported(); 375 376 static bool IsOggEnabled(); 377 static bool IsOpusEnabled(); 378 static bool IsWaveEnabled(); 379 static bool IsWebMEnabled(); 380 381 // Return the frame decode/paint related statistics. GetFrameStatistics()382 FrameStatistics& GetFrameStatistics() { return *mFrameStats; } 383 UpdateReadyState()384 void UpdateReadyState() { 385 MOZ_ASSERT(NS_IsMainThread()); 386 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); 387 GetOwner()->UpdateReadyState(); 388 } 389 NextFrameStatus()390 MediaDecoderOwner::NextFrameStatus NextFrameStatus() const { 391 return mNextFrameStatus; 392 } 393 394 virtual MediaDecoderOwner::NextFrameStatus NextFrameBufferedStatus(); 395 396 RefPtr<GenericPromise> RequestDebugInfo(dom::MediaDecoderDebugInfo& aInfo); 397 398 void GetDebugInfo(dom::MediaDecoderDebugInfo& aInfo); 399 400 protected: 401 virtual ~MediaDecoder(); 402 403 // Called when the first audio and/or video from the media file has been 404 // loaded by the state machine. Call on the main thread only. 405 virtual void FirstFrameLoaded(UniquePtr<MediaInfo> aInfo, 406 MediaDecoderEventVisibility aEventVisibility); 407 408 void SetStateMachineParameters(); 409 410 // Called when MediaDecoder shutdown is finished. Subclasses use this to clean 411 // up internal structures, and unregister potential shutdown blockers when 412 // they're done. 413 virtual void ShutdownInternal(); 414 415 bool IsShutdown() const; 416 417 // Called to notify the decoder that the duration has changed. 418 virtual void DurationChanged(); 419 420 // State-watching manager. 421 WatchManager<MediaDecoder> mWatchManager; 422 ExplicitDuration()423 double ExplicitDuration() { return mExplicitDuration.ref(); } 424 SetExplicitDuration(double aValue)425 void SetExplicitDuration(double aValue) { 426 MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); 427 mExplicitDuration = Some(aValue); 428 429 // We Invoke DurationChanged explicitly, rather than using a watcher, so 430 // that it takes effect immediately, rather than at the end of the current 431 // task. 432 DurationChanged(); 433 } 434 435 virtual void OnPlaybackEvent(MediaPlaybackEvent&& aEvent); 436 437 // Called when the metadata from the media file has been loaded by the 438 // state machine. Call on the main thread only. 439 virtual void MetadataLoaded(UniquePtr<MediaInfo> aInfo, 440 UniquePtr<MetadataTags> aTags, 441 MediaDecoderEventVisibility aEventVisibility); 442 443 void SetLogicalPosition(double aNewPosition); 444 445 /****** 446 * The following members should be accessed with the decoder lock held. 447 ******/ 448 449 // The logical playback position of the media resource in units of 450 // seconds. This corresponds to the "official position" in HTML5. Note that 451 // we need to store this as a double, rather than an int64_t (like 452 // mCurrentPosition), so that |v.currentTime = foo; v.currentTime == foo| 453 // returns true without being affected by rounding errors. 454 double mLogicalPosition; 455 456 // The current playback position of the underlying playback infrastructure. 457 // This corresponds to the "current position" in HTML5. 458 // We allow omx subclasses to substitute an alternative current position for 459 // usage with the audio offload player. CurrentPosition()460 virtual media::TimeUnit CurrentPosition() { return mCurrentPosition.Ref(); } 461 462 already_AddRefed<layers::KnowsCompositor> GetCompositor(); 463 464 // Official duration of the media resource as observed by script. 465 double mDuration; 466 467 /****** 468 * The following member variables can be accessed from any thread. 469 ******/ 470 471 RefPtr<MediaFormatReader> mReader; 472 473 // Amount of buffered data ahead of current time required to consider that 474 // the next frame is available. 475 // An arbitrary value of 250ms is used. 476 static constexpr auto DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED = 477 media::TimeUnit::FromMicroseconds(250000); 478 479 private: 480 // Called when the owner's activity changed. 481 void NotifyCompositor(); 482 483 void OnPlaybackErrorEvent(const MediaResult& aError); 484 485 void OnDecoderDoctorEvent(DecoderDoctorEvent aEvent); 486 OnMediaNotSeekable()487 void OnMediaNotSeekable() { mMediaSeekable = false; } 488 489 void OnNextFrameStatus(MediaDecoderOwner::NextFrameStatus); 490 491 void OnSecondaryVideoContainerInstalled( 492 const RefPtr<VideoFrameContainer>& aSecondaryVideoContainer); 493 494 void OnStoreDecoderBenchmark(const VideoInfo& aInfo); 495 496 void FinishShutdown(); 497 498 void ConnectMirrors(MediaDecoderStateMachine* aObject); 499 void DisconnectMirrors(); 500 501 virtual bool CanPlayThroughImpl() = 0; 502 503 // The state machine object for handling the decoding. It is safe to 504 // call methods of this object from other threads. Its internal data 505 // is synchronised on a monitor. The lifetime of this object is 506 // after mPlayState is LOADING and before mPlayState is SHUTDOWN. It 507 // is safe to access it during this period. 508 // 509 // Explicitly prievate to force access via accessors. 510 RefPtr<MediaDecoderStateMachine> mDecoderStateMachine; 511 512 protected: 513 void NotifyReaderDataArrived(); 514 void DiscardOngoingSeekIfExists(); 515 void CallSeek(const SeekTarget& aTarget); 516 517 // Called by MediaResource when the principal of the resource has 518 // changed. Called on main thread only. 519 virtual void NotifyPrincipalChanged(); 520 521 MozPromiseRequestHolder<SeekPromise> mSeekRequest; 522 523 const char* PlayStateStr(); 524 525 void OnMetadataUpdate(TimedMetadata&& aMetadata); 526 527 // This should only ever be accessed from the main thread. 528 // It is set in the constructor and cleared in Shutdown when the element goes 529 // away. The decoder does not add a reference the element. 530 MediaDecoderOwner* mOwner; 531 532 // The AbstractThread from mOwner. 533 const RefPtr<AbstractThread> mAbstractMainThread; 534 535 // Counters related to decode and presentation of frames. 536 const RefPtr<FrameStatistics> mFrameStats; 537 538 // Store a benchmark of the decoder based on FrameStatistics. 539 RefPtr<DecoderBenchmark> mDecoderBenchmark; 540 541 RefPtr<VideoFrameContainer> mVideoFrameContainer; 542 543 // True if the decoder has been directed to minimize its preroll before 544 // playback starts. After the first time playback starts, we don't attempt 545 // to minimize preroll, as we assume the user is likely to keep playing, 546 // or play the media again. 547 const bool mMinimizePreroll; 548 549 // True if we've already fired metadataloaded. 550 bool mFiredMetadataLoaded; 551 552 // True if the media is seekable (i.e. supports random access). 553 bool mMediaSeekable = true; 554 555 // True if the media is only seekable within its buffered ranges 556 // like WebMs with no cues. 557 bool mMediaSeekableOnlyInBufferedRanges = false; 558 559 // Stores media info, including info of audio tracks and video tracks, should 560 // only be accessed from main thread. 561 UniquePtr<MediaInfo> mInfo; 562 563 // True if the owner element is actually visible to users. 564 bool mIsOwnerInvisible; 565 566 // True if the owner element is connected to a document tree. 567 // https://dom.spec.whatwg.org/#connected 568 bool mIsOwnerConnected; 569 570 // If true, forces the decoder to be considered hidden. 571 bool mForcedHidden; 572 573 // True if the decoder has a suspend taint - meaning suspend-video-decoder is 574 // disabled. 575 bool mHasSuspendTaint; 576 577 MediaDecoderOwner::NextFrameStatus mNextFrameStatus = 578 MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE; 579 580 // A listener to receive metadata updates from MDSM. 581 MediaEventListener mTimedMetadataListener; 582 583 MediaEventListener mMetadataLoadedListener; 584 MediaEventListener mFirstFrameLoadedListener; 585 586 MediaEventListener mOnPlaybackEvent; 587 MediaEventListener mOnPlaybackErrorEvent; 588 MediaEventListener mOnDecoderDoctorEvent; 589 MediaEventListener mOnMediaNotSeekable; 590 MediaEventListener mOnEncrypted; 591 MediaEventListener mOnWaitingForKey; 592 MediaEventListener mOnDecodeWarning; 593 MediaEventListener mOnNextFrameStatus; 594 MediaEventListener mOnSecondaryVideoContainerInstalled; 595 MediaEventListener mOnStoreDecoderBenchmark; 596 597 // True if we have suspended video decoding. 598 bool mIsVideoDecodingSuspended = false; 599 600 protected: 601 // PlaybackRate and pitch preservation status we should start at. 602 double mPlaybackRate; 603 604 // True if the decoder is seeking. 605 Watchable<bool> mLogicallySeeking; 606 607 // Buffered range, mirrored from the reader. 608 Mirror<media::TimeIntervals> mBuffered; 609 610 // NB: Don't use mCurrentPosition directly, but rather CurrentPosition(). 611 Mirror<media::TimeUnit> mCurrentPosition; 612 613 // Duration of the media resource according to the state machine. 614 Mirror<media::NullableTimeUnit> mStateMachineDuration; 615 616 // Used to distinguish whether the audio is producing sound. 617 Mirror<bool> mIsAudioDataAudible; 618 619 // Volume of playback. 0.0 = muted. 1.0 = full volume. 620 Canonical<double> mVolume; 621 622 Canonical<bool> mPreservesPitch; 623 624 Canonical<bool> mLooping; 625 626 Canonical<nsAutoString> mStreamName; 627 628 // The device used with SetSink, or nullptr if no explicit device has been 629 // set. 630 Canonical<RefPtr<AudioDeviceInfo>> mSinkDevice; 631 632 // Set if the decoder is sending video to a secondary container. While set we 633 // should not suspend the decoder. 634 Canonical<RefPtr<VideoFrameContainer>> mSecondaryVideoContainer; 635 636 // Whether this MediaDecoder's output is captured, halted or not captured. 637 // When captured, all decoded data must be played out through mOutputTracks. 638 Canonical<OutputCaptureState> mOutputCaptureState; 639 640 // A dummy track used to access the right MediaTrackGraph instance. Needed 641 // since there's no guarantee that output tracks are present. 642 Canonical<nsMainThreadPtrHandle<SharedDummyTrack>> mOutputDummyTrack; 643 644 // Tracks that, if set, will get data routed through them. 645 Canonical<CopyableTArray<RefPtr<ProcessedMediaTrack>>> mOutputTracks; 646 647 // PrincipalHandle to be used when feeding data into mOutputTracks. 648 Canonical<PrincipalHandle> mOutputPrincipal; 649 650 // Media duration set explicitly by JS. At present, this is only ever present 651 // for MSE. 652 Maybe<double> mExplicitDuration; 653 654 // Set to one of the valid play states. 655 // This can only be changed on the main thread while holding the decoder 656 // monitor. Thus, it can be safely read while holding the decoder monitor 657 // OR on the main thread. 658 Canonical<PlayState> mPlayState; 659 660 // This can only be changed on the main thread. 661 PlayState mNextState = PLAY_STATE_PAUSED; 662 663 // True if the media is same-origin with the element. Data can only be 664 // passed to MediaStreams when this is true. 665 bool mSameOriginMedia; 666 667 // We can allow video decoding in background when we match some special 668 // conditions, eg. when the cursor is hovering over the tab. This observer is 669 // used to listen the related events. 670 RefPtr<BackgroundVideoDecodingPermissionObserver> mVideoDecodingOberver; 671 672 // True if we want to resume video decoding even the media element is in the 673 // background. 674 bool mIsBackgroundVideoDecodingAllowed; 675 676 // True if we want to delay seeking, and and save the latest seeking target to 677 // resume to when we stop delaying seeking. 678 bool mShouldDelaySeek = false; 679 Maybe<SeekTarget> mDelayedSeekTarget; 680 681 public: CanonicalVolume()682 AbstractCanonical<double>* CanonicalVolume() { return &mVolume; } CanonicalPreservesPitch()683 AbstractCanonical<bool>* CanonicalPreservesPitch() { 684 return &mPreservesPitch; 685 } CanonicalLooping()686 AbstractCanonical<bool>* CanonicalLooping() { return &mLooping; } CanonicalStreamName()687 AbstractCanonical<nsAutoString>* CanonicalStreamName() { 688 return &mStreamName; 689 } CanonicalSinkDevice()690 AbstractCanonical<RefPtr<AudioDeviceInfo>>* CanonicalSinkDevice() { 691 return &mSinkDevice; 692 } 693 AbstractCanonical<RefPtr<VideoFrameContainer>>* CanonicalSecondaryVideoContainer()694 CanonicalSecondaryVideoContainer() { 695 return &mSecondaryVideoContainer; 696 } CanonicalOutputCaptureState()697 AbstractCanonical<OutputCaptureState>* CanonicalOutputCaptureState() { 698 return &mOutputCaptureState; 699 } 700 AbstractCanonical<nsMainThreadPtrHandle<SharedDummyTrack>>* CanonicalOutputDummyTrack()701 CanonicalOutputDummyTrack() { 702 return &mOutputDummyTrack; 703 } 704 AbstractCanonical<CopyableTArray<RefPtr<ProcessedMediaTrack>>>* CanonicalOutputTracks()705 CanonicalOutputTracks() { 706 return &mOutputTracks; 707 } CanonicalOutputPrincipal()708 AbstractCanonical<PrincipalHandle>* CanonicalOutputPrincipal() { 709 return &mOutputPrincipal; 710 } CanonicalPlayState()711 AbstractCanonical<PlayState>* CanonicalPlayState() { return &mPlayState; } 712 713 void UpdateTelemetryHelperBasedOnPlayState(PlayState aState) const; 714 715 TelemetryProbesReporter::Visibility OwnerVisibility() const; 716 717 // Those methods exist to report telemetry related metrics. 718 double GetTotalVideoPlayTimeInSeconds() const; 719 double GetVisibleVideoPlayTimeInSeconds() const; 720 double GetInvisibleVideoPlayTimeInSeconds() const; 721 double GetVideoDecodeSuspendedTimeInSeconds() const; 722 double GetTotalAudioPlayTimeInSeconds() const; 723 double GetAudiblePlayTimeInSeconds() const; 724 double GetInaudiblePlayTimeInSeconds() const; 725 double GetMutedPlayTimeInSeconds() const; 726 727 private: 728 /** 729 * This enum describes the reason why we need to update the logical position. 730 * ePeriodicUpdate : the position grows periodically during playback 731 * eSeamlessLoopingSeeking : the position changes due to demuxer level seek. 732 * eOther : due to normal seeking or other attributes changes, eg. playstate 733 */ 734 enum class PositionUpdate { 735 ePeriodicUpdate, 736 eSeamlessLoopingSeeking, 737 eOther, 738 }; 739 PositionUpdate GetPositionUpdateReason(double aPrevPos, double aCurPos) const; 740 741 // Notify owner when the audible state changed 742 void NotifyAudibleStateChanged(); 743 744 void NotifyVolumeChanged(); 745 746 bool mTelemetryReported; 747 const MediaContainerType mContainerType; 748 bool mCanPlayThrough = false; 749 750 UniquePtr<TelemetryProbesReporter> mTelemetryProbesReporter; 751 }; 752 753 typedef MozPromise<mozilla::dom::MediaMemoryInfo, nsresult, true> 754 MediaMemoryPromise; 755 756 RefPtr<MediaMemoryPromise> GetMediaMemorySizes(); 757 758 } // namespace mozilla 759 760 #endif 761