1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 /* 7 8 Each media element for a media file has one thread called the "audio thread". 9 10 The audio thread writes the decoded audio data to the audio 11 hardware. This is done in a separate thread to ensure that the 12 audio hardware gets a constant stream of data without 13 interruption due to decoding or display. At some point 14 AudioStream will be refactored to have a callback interface 15 where it asks for data and this thread will no longer be 16 needed. 17 18 The element/state machine also has a TaskQueue which runs in a 19 SharedThreadPool that is shared with all other elements/decoders. The state 20 machine dispatches tasks to this to call into the MediaDecoderReader to 21 request decoded audio or video data. The Reader will callback with decoded 22 sampled when it has them available, and the state machine places the decoded 23 samples into its queues for the consuming threads to pull from. 24 25 The MediaDecoderReader can choose to decode asynchronously, or synchronously 26 and return requested samples synchronously inside it's Request*Data() 27 functions via callback. Asynchronous decoding is preferred, and should be 28 used for any new readers. 29 30 Synchronisation of state between the thread is done via a monitor owned 31 by MediaDecoder. 32 33 The lifetime of the audio thread is controlled by the state machine when 34 it runs on the shared state machine thread. When playback needs to occur 35 the audio thread is created and an event dispatched to run it. The audio 36 thread exits when audio playback is completed or no longer required. 37 38 A/V synchronisation is handled by the state machine. It examines the audio 39 playback time and compares this to the next frame in the queue of video 40 frames. If it is time to play the video frame it is then displayed, otherwise 41 it schedules the state machine to run again at the time of the next frame. 42 43 Frame skipping is done in the following ways: 44 45 1) The state machine will skip all frames in the video queue whose 46 display time is less than the current audio time. This ensures 47 the correct frame for the current time is always displayed. 48 49 2) The decode tasks will stop decoding interframes and read to the 50 next keyframe if it determines that decoding the remaining 51 interframes will cause playback issues. It detects this by: 52 a) If the amount of audio data in the audio queue drops 53 below a threshold whereby audio may start to skip. 54 b) If the video queue drops below a threshold where it 55 will be decoding video data that won't be displayed due 56 to the decode thread dropping the frame immediately. 57 TODO: In future we should only do this when the Reader is decoding 58 synchronously. 59 60 When hardware accelerated graphics is not available, YCbCr conversion 61 is done on the decode task queue when video frames are decoded. 62 63 The decode task queue pushes decoded audio and videos frames into two 64 separate queues - one for audio and one for video. These are kept 65 separate to make it easy to constantly feed audio data to the audio 66 hardware while allowing frame skipping of video data. These queues are 67 threadsafe, and neither the decode, audio, or state machine should 68 be able to monopolize them, and cause starvation of the other threads. 69 70 Both queues are bounded by a maximum size. When this size is reached 71 the decode tasks will no longer request video or audio depending on the 72 queue that has reached the threshold. If both queues are full, no more 73 decode tasks will be dispatched to the decode task queue, so other 74 decoders will have an opportunity to run. 75 76 During playback the audio thread will be idle (via a Wait() on the 77 monitor) if the audio queue is empty. Otherwise it constantly pops 78 audio data off the queue and plays it with a blocking write to the audio 79 hardware (via AudioStream). 80 81 */ 82 #if !defined(MediaDecoderStateMachine_h__) 83 # define MediaDecoderStateMachine_h__ 84 85 # include "AudioDeviceInfo.h" 86 # include "ImageContainer.h" 87 # include "MediaDecoder.h" 88 # include "MediaDecoderOwner.h" 89 # include "MediaEventSource.h" 90 # include "MediaFormatReader.h" 91 # include "MediaMetadataManager.h" 92 # include "MediaQueue.h" 93 # include "MediaSink.h" 94 # include "MediaStatistics.h" 95 # include "MediaTimer.h" 96 # include "SeekJob.h" 97 # include "mozilla/Attributes.h" 98 # include "mozilla/ReentrantMonitor.h" 99 # include "mozilla/StateMirroring.h" 100 # include "mozilla/dom/MediaDebugInfoBinding.h" 101 # include "nsThreadUtils.h" 102 103 namespace mozilla { 104 105 class AbstractThread; 106 class AudioSegment; 107 class DecodedStream; 108 class DOMMediaStream; 109 class ReaderProxy; 110 class TaskQueue; 111 112 extern LazyLogModule gMediaDecoderLog; 113 114 struct MediaPlaybackEvent { 115 enum EventType { 116 PlaybackStarted, 117 PlaybackStopped, 118 PlaybackProgressed, 119 PlaybackEnded, 120 SeekStarted, 121 Invalidate, 122 EnterVideoSuspend, 123 ExitVideoSuspend, 124 StartVideoSuspendTimer, 125 CancelVideoSuspendTimer, 126 VideoOnlySeekBegin, 127 VideoOnlySeekCompleted, 128 } mType; 129 130 using DataType = Variant<Nothing, int64_t>; 131 DataType mData; 132 MediaPlaybackEventMediaPlaybackEvent133 MOZ_IMPLICIT MediaPlaybackEvent(EventType aType) 134 : mType(aType), mData(Nothing{}) {} 135 136 template <typename T> MediaPlaybackEventMediaPlaybackEvent137 MediaPlaybackEvent(EventType aType, T&& aArg) 138 : mType(aType), mData(std::forward<T>(aArg)) {} 139 }; 140 141 enum class VideoDecodeMode : uint8_t { Normal, Suspend }; 142 143 DDLoggedTypeDeclName(MediaDecoderStateMachine); 144 145 /* 146 The state machine class. This manages the decoding and seeking in the 147 MediaDecoderReader on the decode task queue, and A/V sync on the shared 148 state machine thread, and controls the audio "push" thread. 149 150 All internal state is synchronised via the decoder monitor. State changes 151 are propagated by scheduling the state machine to run another cycle on the 152 shared state machine thread. 153 154 See MediaDecoder.h for more details. 155 */ 156 class MediaDecoderStateMachine 157 : public DecoderDoctorLifeLogger<MediaDecoderStateMachine> { 158 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderStateMachine) 159 160 using TrackSet = MediaFormatReader::TrackSet; 161 162 public: 163 typedef MediaDecoderOwner::NextFrameStatus NextFrameStatus; 164 typedef mozilla::layers::ImageContainer::FrameID FrameID; 165 MediaDecoderStateMachine(MediaDecoder* aDecoder, MediaFormatReader* aReader); 166 167 nsresult Init(MediaDecoder* aDecoder); 168 169 // Enumeration for the valid decoding states 170 enum State { 171 DECODER_STATE_DECODING_METADATA, 172 DECODER_STATE_DORMANT, 173 DECODER_STATE_DECODING_FIRSTFRAME, 174 DECODER_STATE_DECODING, 175 DECODER_STATE_LOOPING_DECODING, 176 DECODER_STATE_SEEKING_ACCURATE, 177 DECODER_STATE_SEEKING_FROMDORMANT, 178 DECODER_STATE_SEEKING_NEXTFRAMESEEKING, 179 DECODER_STATE_SEEKING_VIDEOONLY, 180 DECODER_STATE_BUFFERING, 181 DECODER_STATE_COMPLETED, 182 DECODER_STATE_SHUTDOWN 183 }; 184 185 // Returns the state machine task queue. OwnerThread()186 TaskQueue* OwnerThread() const { return mTaskQueue; } 187 188 RefPtr<GenericPromise> RequestDebugInfo( 189 dom::MediaDecoderStateMachineDebugInfo& aInfo); 190 191 // Seeks to the decoder to aTarget asynchronously. 192 RefPtr<MediaDecoder::SeekPromise> InvokeSeek(const SeekTarget& aTarget); 193 DispatchSetPlaybackRate(double aPlaybackRate)194 void DispatchSetPlaybackRate(double aPlaybackRate) { 195 OwnerThread()->DispatchStateChange(NewRunnableMethod<double>( 196 "MediaDecoderStateMachine::SetPlaybackRate", this, 197 &MediaDecoderStateMachine::SetPlaybackRate, aPlaybackRate)); 198 } 199 200 RefPtr<ShutdownPromise> BeginShutdown(); 201 202 // Set the media fragment end time. DispatchSetFragmentEndTime(const media::TimeUnit & aEndTime)203 void DispatchSetFragmentEndTime(const media::TimeUnit& aEndTime) { 204 RefPtr<MediaDecoderStateMachine> self = this; 205 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction( 206 "MediaDecoderStateMachine::DispatchSetFragmentEndTime", 207 [self, aEndTime]() { 208 // A negative number means we don't have a fragment end time at all. 209 self->mFragmentEndTime = aEndTime >= media::TimeUnit::Zero() 210 ? aEndTime 211 : media::TimeUnit::Invalid(); 212 }); 213 nsresult rv = OwnerThread()->Dispatch(r.forget()); 214 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); 215 Unused << rv; 216 } 217 DispatchCanPlayThrough(bool aCanPlayThrough)218 void DispatchCanPlayThrough(bool aCanPlayThrough) { 219 RefPtr<MediaDecoderStateMachine> self = this; 220 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction( 221 "MediaDecoderStateMachine::DispatchCanPlayThrough", 222 [self, aCanPlayThrough]() { self->mCanPlayThrough = aCanPlayThrough; }); 223 OwnerThread()->DispatchStateChange(r.forget()); 224 } 225 DispatchIsLiveStream(bool aIsLiveStream)226 void DispatchIsLiveStream(bool aIsLiveStream) { 227 RefPtr<MediaDecoderStateMachine> self = this; 228 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction( 229 "MediaDecoderStateMachine::DispatchIsLiveStream", 230 [self, aIsLiveStream]() { self->mIsLiveStream = aIsLiveStream; }); 231 OwnerThread()->DispatchStateChange(r.forget()); 232 } 233 TimedMetadataEvent()234 TimedMetadataEventSource& TimedMetadataEvent() { 235 return mMetadataManager.TimedMetadataEvent(); 236 } 237 238 MediaEventSource<void>& OnMediaNotSeekable() const; 239 240 MediaEventSourceExc<UniquePtr<MediaInfo>, UniquePtr<MetadataTags>, 241 MediaDecoderEventVisibility>& MetadataLoadedEvent()242 MetadataLoadedEvent() { 243 return mMetadataLoadedEvent; 244 } 245 246 MediaEventSourceExc<UniquePtr<MediaInfo>, MediaDecoderEventVisibility>& FirstFrameLoadedEvent()247 FirstFrameLoadedEvent() { 248 return mFirstFrameLoadedEvent; 249 } 250 OnPlaybackEvent()251 MediaEventSource<MediaPlaybackEvent>& OnPlaybackEvent() { 252 return mOnPlaybackEvent; 253 } OnPlaybackErrorEvent()254 MediaEventSource<MediaResult>& OnPlaybackErrorEvent() { 255 return mOnPlaybackErrorEvent; 256 } 257 OnDecoderDoctorEvent()258 MediaEventSource<DecoderDoctorEvent>& OnDecoderDoctorEvent() { 259 return mOnDecoderDoctorEvent; 260 } 261 OnNextFrameStatus()262 MediaEventSource<NextFrameStatus>& OnNextFrameStatus() { 263 return mOnNextFrameStatus; 264 } 265 266 MediaEventSourceExc<RefPtr<VideoFrameContainer>>& OnSecondaryVideoContainerInstalled()267 OnSecondaryVideoContainerInstalled() { 268 return mOnSecondaryVideoContainerInstalled; 269 } 270 271 size_t SizeOfVideoQueue() const; 272 273 size_t SizeOfAudioQueue() const; 274 275 // Sets the video decode mode. Used by the suspend-video-decoder feature. 276 void SetVideoDecodeMode(VideoDecodeMode aMode); 277 278 RefPtr<GenericPromise> InvokeSetSink(const RefPtr<AudioDeviceInfo>& aSink); 279 280 void InvokeSuspendMediaSink(); 281 void InvokeResumeMediaSink(); 282 283 private: 284 class StateObject; 285 class DecodeMetadataState; 286 class DormantState; 287 class DecodingFirstFrameState; 288 class DecodingState; 289 class LoopingDecodingState; 290 class SeekingState; 291 class AccurateSeekingState; 292 class NextFrameSeekingState; 293 class NextFrameSeekingFromDormantState; 294 class VideoOnlySeekingState; 295 class BufferingState; 296 class CompletedState; 297 class ShutdownState; 298 299 static const char* ToStateStr(State aState); 300 const char* ToStateStr(); 301 302 void GetDebugInfo(dom::MediaDecoderStateMachineDebugInfo& aInfo); 303 304 // Functions used by assertions to ensure we're calling things 305 // on the appropriate threads. 306 bool OnTaskQueue() const; 307 308 // Initialization that needs to happen on the task queue. This is the first 309 // task that gets run on the task queue, and is dispatched from the MDSM 310 // constructor immediately after the task queue is created. 311 void InitializationTask(MediaDecoder* aDecoder); 312 313 RefPtr<MediaDecoder::SeekPromise> Seek(const SeekTarget& aTarget); 314 315 RefPtr<ShutdownPromise> Shutdown(); 316 317 RefPtr<ShutdownPromise> FinishShutdown(); 318 319 // Update the playback position. This can result in a timeupdate event 320 // and an invalidate of the frame being dispatched asynchronously if 321 // there is no such event currently queued. 322 // Only called on the decoder thread. Must be called with 323 // the decode monitor held. 324 void UpdatePlaybackPosition(const media::TimeUnit& aTime); 325 HasAudio()326 bool HasAudio() const { return mInfo.ref().HasAudio(); } HasVideo()327 bool HasVideo() const { return mInfo.ref().HasVideo(); } Info()328 const MediaInfo& Info() const { return mInfo.ref(); } 329 330 // Schedules the shared state machine thread to run the state machine. 331 void ScheduleStateMachine(); 332 333 // Invokes ScheduleStateMachine to run in |aTime|, 334 // unless it's already scheduled to run earlier, in which case the 335 // request is discarded. 336 void ScheduleStateMachineIn(const media::TimeUnit& aTime); 337 338 bool HaveEnoughDecodedAudio() const; 339 bool HaveEnoughDecodedVideo() const; 340 341 // The check is used to store more video frames than usual when playing 4K+ 342 // video. 343 bool IsVideoDataEnoughComparedWithAudio() const; 344 345 // Returns true if we're currently playing. The decoder monitor must 346 // be held. 347 bool IsPlaying() const; 348 349 // Sets mMediaSeekable to false. 350 void SetMediaNotSeekable(); 351 352 // Resets all states related to decoding and aborts all pending requests 353 // to the decoders. 354 void ResetDecode(const TrackSet& aTracks = TrackSet(TrackInfo::kAudioTrack, 355 TrackInfo::kVideoTrack)); 356 357 void SetVideoDecodeModeInternal(VideoDecodeMode aMode); 358 359 // Set new sink device and restart MediaSink if playback is started. 360 // Returned promise will be resolved with true if the playback is 361 // started and false if playback is stopped after setting the new sink. 362 // Returned promise will be rejected with value NS_ERROR_ABORT 363 // if the action fails or it is not supported. 364 // If there are multiple pending requests only the last one will be 365 // executed, for all previous requests the promise will be resolved 366 // with true or false similar to above. 367 RefPtr<GenericPromise> SetSink(const RefPtr<AudioDeviceInfo>& aDevice); 368 369 // Shutdown MediaSink on suspend to clean up resources. 370 void SuspendMediaSink(); 371 // Create a new MediaSink, it must have been stopped first. 372 void ResumeMediaSink(); 373 374 protected: 375 virtual ~MediaDecoderStateMachine(); 376 377 void BufferedRangeUpdated(); 378 379 void ReaderSuspendedChanged(); 380 381 // Inserts a sample into the Audio/Video queue. 382 // aSample must not be null. 383 void PushAudio(AudioData* aSample); 384 void PushVideo(VideoData* aSample); 385 386 void OnAudioPopped(const RefPtr<AudioData>& aSample); 387 void OnVideoPopped(const RefPtr<VideoData>& aSample); 388 389 void AudioAudibleChanged(bool aAudible); 390 391 void VolumeChanged(); 392 void SetPlaybackRate(double aPlaybackRate); 393 void PreservesPitchChanged(); 394 void LoopingChanged(); 395 void StreamNameChanged(); 396 void UpdateSecondaryVideoContainer(); 397 void UpdateOutputCaptured(); 398 void OutputTracksChanged(); 399 void OutputPrincipalChanged(); 400 AudioQueue()401 MediaQueue<AudioData>& AudioQueue() { return mAudioQueue; } VideoQueue()402 MediaQueue<VideoData>& VideoQueue() { return mVideoQueue; } 403 AudioQueue()404 const MediaQueue<AudioData>& AudioQueue() const { return mAudioQueue; } VideoQueue()405 const MediaQueue<VideoData>& VideoQueue() const { return mVideoQueue; } 406 407 // True if we are low in decoded audio/video data. 408 // May not be invoked when mReader->UseBufferingHeuristics() is false. 409 bool HasLowDecodedData(); 410 411 bool HasLowDecodedAudio(); 412 413 bool HasLowDecodedVideo(); 414 415 bool OutOfDecodedAudio(); 416 OutOfDecodedVideo()417 bool OutOfDecodedVideo() { 418 MOZ_ASSERT(OnTaskQueue()); 419 return IsVideoDecoding() && VideoQueue().GetSize() <= 1; 420 } 421 422 // Returns true if we're running low on buffered data. 423 bool HasLowBufferedData(); 424 425 // Returns true if we have less than aThreshold of buffered data available. 426 bool HasLowBufferedData(const media::TimeUnit& aThreshold); 427 428 // Return the current time, either the audio clock if available (if the media 429 // has audio, and the playback is possible), or a clock for the video. 430 // Called on the state machine thread. 431 // If aTimeStamp is non-null, set *aTimeStamp to the TimeStamp corresponding 432 // to the returned stream time. 433 media::TimeUnit GetClock(TimeStamp* aTimeStamp = nullptr) const; 434 435 // Update only the state machine's current playback position (and duration, 436 // if unknown). Does not update the playback position on the decoder or 437 // media element -- use UpdatePlaybackPosition for that. Called on the state 438 // machine thread, caller must hold the decoder lock. 439 void UpdatePlaybackPositionInternal(const media::TimeUnit& aTime); 440 441 // Update playback position and trigger next update by default time period. 442 // Called on the state machine thread. 443 void UpdatePlaybackPositionPeriodically(); 444 445 MediaSink* CreateAudioSink(); 446 447 // Always create mediasink which contains an AudioSink or DecodedStream 448 // inside. 449 already_AddRefed<MediaSink> CreateMediaSink(); 450 451 // Stops the media sink and shut it down. 452 // The decoder monitor must be held with exactly one lock count. 453 // Called on the state machine thread. 454 void StopMediaSink(); 455 456 // Create and start the media sink. 457 // The decoder monitor must be held with exactly one lock count. 458 // Called on the state machine thread. 459 // If start fails an NS_ERROR_FAILURE is returned. 460 nsresult StartMediaSink(); 461 462 // Notification method invoked when mPlayState changes. 463 void PlayStateChanged(); 464 465 // Notification method invoked when mIsVisible changes. 466 void VisibilityChanged(); 467 468 // Sets internal state which causes playback of media to pause. 469 // The decoder monitor must be held. 470 void StopPlayback(); 471 472 // If the conditions are right, sets internal state which causes playback 473 // of media to begin or resume. 474 // Must be called with the decode monitor held. 475 void MaybeStartPlayback(); 476 477 // Moves the decoder into the shutdown state, and dispatches an error 478 // event to the media element. This begins shutting down the decoder. 479 // The decoder monitor must be held. This is only called on the 480 // decode thread. 481 void DecodeError(const MediaResult& aError); 482 483 void EnqueueFirstFrameLoadedEvent(); 484 485 // Start a task to decode audio. 486 void RequestAudioData(); 487 488 // Start a task to decode video. 489 // @param aRequestNextVideoKeyFrame 490 // If aRequestNextKeyFrame is true, will request data for the next keyframe 491 // after aCurrentTime. 492 void RequestVideoData(const media::TimeUnit& aCurrentTime, 493 bool aRequestNextKeyFrame = false); 494 495 void WaitForData(MediaData::Type aType); 496 IsRequestingAudioData()497 bool IsRequestingAudioData() const { return mAudioDataRequest.Exists(); } IsRequestingVideoData()498 bool IsRequestingVideoData() const { return mVideoDataRequest.Exists(); } IsWaitingAudioData()499 bool IsWaitingAudioData() const { return mAudioWaitRequest.Exists(); } IsWaitingVideoData()500 bool IsWaitingVideoData() const { return mVideoWaitRequest.Exists(); } 501 502 // Returns the "media time". This is the absolute time which the media 503 // playback has reached. i.e. this returns values in the range 504 // [mStartTime, mEndTime], and mStartTime will not be 0 if the media does 505 // not start at 0. Note this is different than the "current playback 506 // position", which is in the range [0,duration]. GetMediaTime()507 media::TimeUnit GetMediaTime() const { 508 MOZ_ASSERT(OnTaskQueue()); 509 return mCurrentPosition; 510 } 511 512 // Returns an upper bound on the number of microseconds of audio that is 513 // decoded and playable. This is the sum of the number of usecs of audio which 514 // is decoded and in the reader's audio queue, and the usecs of unplayed audio 515 // which has been pushed to the audio hardware for playback. Note that after 516 // calling this, the audio hardware may play some of the audio pushed to 517 // hardware, so this can only be used as a upper bound. The decoder monitor 518 // must be held when calling this. Called on the decode thread. 519 media::TimeUnit GetDecodedAudioDuration() const; 520 521 void FinishDecodeFirstFrame(); 522 523 // Performs one "cycle" of the state machine. 524 void RunStateMachine(); 525 526 bool IsStateMachineScheduled() const; 527 528 // These return true if the respective stream's decode has not yet reached 529 // the end of stream. 530 bool IsAudioDecoding(); 531 bool IsVideoDecoding(); 532 533 private: 534 // Resolved by the MediaSink to signal that all audio/video outstanding 535 // work is complete and identify which part(a/v) of the sink is shutting down. 536 void OnMediaSinkAudioComplete(); 537 void OnMediaSinkVideoComplete(); 538 539 // Rejected by the MediaSink to signal errors for audio/video. 540 void OnMediaSinkAudioError(nsresult aResult); 541 void OnMediaSinkVideoError(); 542 543 void* const mDecoderID; 544 const RefPtr<AbstractThread> mAbstractMainThread; 545 const RefPtr<FrameStatistics> mFrameStats; 546 const RefPtr<VideoFrameContainer> mVideoFrameContainer; 547 548 // Task queue for running the state machine. 549 RefPtr<TaskQueue> mTaskQueue; 550 551 // State-watching manager. 552 WatchManager<MediaDecoderStateMachine> mWatchManager; 553 554 // True if we've dispatched a task to run the state machine but the task has 555 // yet to run. 556 bool mDispatchedStateMachine; 557 558 // Used to dispatch another round schedule with specific target time. 559 DelayedScheduler mDelayedScheduler; 560 561 // Queue of audio frames. This queue is threadsafe, and is accessed from 562 // the audio, decoder, state machine, and main threads. 563 MediaQueue<AudioData> mAudioQueue; 564 // Queue of video frames. This queue is threadsafe, and is accessed from 565 // the decoder, state machine, and main threads. 566 MediaQueue<VideoData> mVideoQueue; 567 568 UniquePtr<StateObject> mStateObj; 569 Duration()570 media::TimeUnit Duration() const { 571 MOZ_ASSERT(OnTaskQueue()); 572 return mDuration.Ref().ref(); 573 } 574 575 // FrameID which increments every time a frame is pushed to our queue. 576 FrameID mCurrentFrameID; 577 578 // Media Fragment end time. 579 media::TimeUnit mFragmentEndTime = media::TimeUnit::Invalid(); 580 581 // The media sink resource. Used on the state machine thread. 582 RefPtr<MediaSink> mMediaSink; 583 584 const RefPtr<ReaderProxy> mReader; 585 586 // The end time of the last audio frame that's been pushed onto the media sink 587 // in microseconds. This will approximately be the end time 588 // of the audio stream, unless another frame is pushed to the hardware. 589 media::TimeUnit AudioEndTime() const; 590 591 // The end time of the last rendered video frame that's been sent to 592 // compositor. 593 media::TimeUnit VideoEndTime() const; 594 595 // The end time of the last decoded audio frame. This signifies the end of 596 // decoded audio data. Used to check if we are low in decoded data. 597 media::TimeUnit mDecodedAudioEndTime; 598 599 // The end time of the last decoded video frame. Used to check if we are low 600 // on decoded video data. 601 media::TimeUnit mDecodedVideoEndTime; 602 603 // Playback rate. 1.0 : normal speed, 0.5 : two times slower. 604 double mPlaybackRate; 605 606 // If we've got more than this number of decoded video frames waiting in 607 // the video queue, we will not decode any more video frames until some have 608 // been consumed by the play state machine thread. 609 // Must hold monitor. 610 uint32_t GetAmpleVideoFrames() const; 611 612 // Our "ample" audio threshold. Once we've this much audio decoded, we 613 // pause decoding. 614 media::TimeUnit mAmpleAudioThreshold; 615 616 // Only one of a given pair of ({Audio,Video}DataPromise, WaitForDataPromise) 617 // should exist at any given moment. 618 using AudioDataPromise = MediaFormatReader::AudioDataPromise; 619 using VideoDataPromise = MediaFormatReader::VideoDataPromise; 620 using WaitForDataPromise = MediaFormatReader::WaitForDataPromise; 621 MozPromiseRequestHolder<AudioDataPromise> mAudioDataRequest; 622 MozPromiseRequestHolder<VideoDataPromise> mVideoDataRequest; 623 MozPromiseRequestHolder<WaitForDataPromise> mAudioWaitRequest; 624 MozPromiseRequestHolder<WaitForDataPromise> mVideoWaitRequest; 625 626 const char* AudioRequestStatus() const; 627 const char* VideoRequestStatus() const; 628 629 void OnSuspendTimerResolved(); 630 void CancelSuspendTimer(); 631 632 bool IsInSeamlessLooping() const; 633 634 bool mCanPlayThrough = false; 635 636 bool mIsLiveStream = false; 637 638 // True if all audio frames are already rendered. 639 bool mAudioCompleted = false; 640 641 // True if all video frames are already rendered. 642 bool mVideoCompleted = false; 643 644 // True if we should not decode/preroll unnecessary samples, unless we're 645 // played. "Prerolling" in this context refers to when we decode and 646 // buffer decoded samples in advance of when they're needed for playback. 647 // This flag is set for preload=metadata media, and means we won't 648 // decode more than the first video frame and first block of audio samples 649 // for that media when we startup, or after a seek. When Play() is called, 650 // we reset this flag, as we assume the user is playing the media, so 651 // prerolling is appropriate then. This flag is used to reduce the overhead 652 // of prerolling samples for media elements that may not play, both 653 // memory and CPU overhead. 654 bool mMinimizePreroll; 655 656 // Stores presentation info required for playback. 657 Maybe<MediaInfo> mInfo; 658 659 mozilla::MediaMetadataManager mMetadataManager; 660 661 // True if we've decoded first frames (thus having the start time) and 662 // notified the FirstFrameLoaded event. Note we can't initiate seek until the 663 // start time is known which happens when the first frames are decoded or we 664 // are playing an MSE stream (the start time is always assumed 0). 665 bool mSentFirstFrameLoadedEvent; 666 667 // True if video decoding is suspended. 668 bool mVideoDecodeSuspended; 669 670 // True if the media is seekable (i.e. supports random access). 671 bool mMediaSeekable = true; 672 673 // True if the media is seekable only in buffered ranges. 674 bool mMediaSeekableOnlyInBufferedRanges = false; 675 676 // Track enabling video decode suspension via timer 677 DelayedScheduler mVideoDecodeSuspendTimer; 678 679 // Track the current video decode mode. 680 VideoDecodeMode mVideoDecodeMode; 681 682 // Track the complete & error for audio/video separately 683 MozPromiseRequestHolder<MediaSink::EndedPromise> mMediaSinkAudioEndedPromise; 684 MozPromiseRequestHolder<MediaSink::EndedPromise> mMediaSinkVideoEndedPromise; 685 686 MediaEventListener mAudioQueueListener; 687 MediaEventListener mVideoQueueListener; 688 MediaEventListener mAudibleListener; 689 MediaEventListener mOnMediaNotSeekable; 690 691 MediaEventProducerExc<UniquePtr<MediaInfo>, UniquePtr<MetadataTags>, 692 MediaDecoderEventVisibility> 693 mMetadataLoadedEvent; 694 MediaEventProducerExc<UniquePtr<MediaInfo>, MediaDecoderEventVisibility> 695 mFirstFrameLoadedEvent; 696 697 MediaEventProducer<MediaPlaybackEvent> mOnPlaybackEvent; 698 MediaEventProducer<MediaResult> mOnPlaybackErrorEvent; 699 700 MediaEventProducer<DecoderDoctorEvent> mOnDecoderDoctorEvent; 701 702 MediaEventProducer<NextFrameStatus> mOnNextFrameStatus; 703 704 MediaEventProducerExc<RefPtr<VideoFrameContainer>> 705 mOnSecondaryVideoContainerInstalled; 706 707 const bool mIsMSE; 708 709 bool mSeamlessLoopingAllowed; 710 711 // If media was in looping and had reached to the end before, then we need 712 // to adjust sample time from clock time to media time. 713 void AdjustByLooping(media::TimeUnit& aTime) const; 714 Maybe<media::TimeUnit> mAudioDecodedDuration; 715 716 // Current playback position in the stream in bytes. 717 int64_t mPlaybackOffset = 0; 718 719 private: 720 // The buffered range. Mirrored from the decoder thread. 721 Mirror<media::TimeIntervals> mBuffered; 722 723 // The current play state, mirrored from the main thread. 724 Mirror<MediaDecoder::PlayState> mPlayState; 725 726 // Volume of playback. 0.0 = muted. 1.0 = full volume. 727 Mirror<double> mVolume; 728 729 // Pitch preservation for the playback rate. 730 Mirror<bool> mPreservesPitch; 731 732 // Whether to seek back to the start of the media resource 733 // upon reaching the end. 734 Mirror<bool> mLooping; 735 736 // Audio stream name 737 Mirror<nsAutoString> mStreamName; 738 739 // The device used with SetSink, or nullptr if no explicit device has been 740 // set. 741 Mirror<RefPtr<AudioDeviceInfo>> mSinkDevice; 742 743 // Set if the decoder is sending video to a secondary container. While set we 744 // should not suspend the decoder. 745 Mirror<RefPtr<VideoFrameContainer>> mSecondaryVideoContainer; 746 747 // Whether all output should be captured into mOutputTracks, halted, or not 748 // captured. 749 Mirror<MediaDecoder::OutputCaptureState> mOutputCaptureState; 750 751 // A dummy track used to access the right MediaTrackGraph instance. Needed 752 // since there's no guarantee that output tracks are present. 753 Mirror<nsMainThreadPtrHandle<SharedDummyTrack>> mOutputDummyTrack; 754 755 // Tracks to capture data into. 756 Mirror<CopyableTArray<RefPtr<ProcessedMediaTrack>>> mOutputTracks; 757 758 // PrincipalHandle to feed with data captured into mOutputTracks. 759 Mirror<PrincipalHandle> mOutputPrincipal; 760 761 Canonical<CopyableTArray<RefPtr<ProcessedMediaTrack>>> mCanonicalOutputTracks; 762 Canonical<PrincipalHandle> mCanonicalOutputPrincipal; 763 764 // Duration of the media. This is guaranteed to be non-null after we finish 765 // decoding the first frame. 766 Canonical<media::NullableTimeUnit> mDuration; 767 768 // The time of the current frame, corresponding to the "current 769 // playback position" in HTML5. This is referenced from 0, which is the 770 // initial playback position. 771 Canonical<media::TimeUnit> mCurrentPosition; 772 773 // Used to distinguish whether the audio is producing sound. 774 Canonical<bool> mIsAudioDataAudible; 775 776 // Track when MediaSink is supsended. When that happens some actions are 777 // restricted like starting the sink or changing sink id. The flag is valid 778 // after Initialization. TaskQueue thread only. 779 bool mIsMediaSinkSuspended = false; 780 781 public: 782 AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() const; 783 784 AbstractCanonical<CopyableTArray<RefPtr<ProcessedMediaTrack>>>* CanonicalOutputTracks()785 CanonicalOutputTracks() { 786 return &mCanonicalOutputTracks; 787 } CanonicalOutputPrincipal()788 AbstractCanonical<PrincipalHandle>* CanonicalOutputPrincipal() { 789 return &mCanonicalOutputPrincipal; 790 } CanonicalDuration()791 AbstractCanonical<media::NullableTimeUnit>* CanonicalDuration() { 792 return &mDuration; 793 } CanonicalCurrentPosition()794 AbstractCanonical<media::TimeUnit>* CanonicalCurrentPosition() { 795 return &mCurrentPosition; 796 } CanonicalIsAudioDataAudible()797 AbstractCanonical<bool>* CanonicalIsAudioDataAudible() { 798 return &mIsAudioDataAudible; 799 } 800 }; 801 802 } // namespace mozilla 803 804 #endif 805