1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #if !defined(MediaFormatReader_h_) 8 # define MediaFormatReader_h_ 9 10 # include "mozilla/Atomics.h" 11 # include "mozilla/Maybe.h" 12 # include "mozilla/Mutex.h" 13 # include "mozilla/StateMirroring.h" 14 # include "mozilla/StaticPrefs_media.h" 15 # include "mozilla/TaskQueue.h" 16 # include "mozilla/dom/MediaDebugInfoBinding.h" 17 18 # include "FrameStatistics.h" 19 # include "MediaEventSource.h" 20 # include "MediaDataDemuxer.h" 21 # include "MediaMetadataManager.h" 22 # include "MediaPromiseDefs.h" 23 # include "PDMFactory.h" 24 # include "SeekTarget.h" 25 26 namespace mozilla { 27 28 class CDMProxy; 29 class GMPCrashHelper; 30 class MediaResource; 31 class VideoFrameContainer; 32 33 struct WaitForDataRejectValue { 34 enum Reason { SHUTDOWN, CANCELED }; 35 WaitForDataRejectValueWaitForDataRejectValue36 WaitForDataRejectValue(MediaData::Type aType, Reason aReason) 37 : mType(aType), mReason(aReason) {} 38 MediaData::Type mType; 39 Reason mReason; 40 }; 41 42 struct SeekRejectValue { SeekRejectValueSeekRejectValue43 MOZ_IMPLICIT SeekRejectValue(const MediaResult& aError) 44 : mType(MediaData::Type::NULL_DATA), mError(aError) {} SeekRejectValueSeekRejectValue45 MOZ_IMPLICIT SeekRejectValue(nsresult aResult) 46 : mType(MediaData::Type::NULL_DATA), mError(aResult) {} SeekRejectValueSeekRejectValue47 SeekRejectValue(MediaData::Type aType, const MediaResult& aError) 48 : mType(aType), mError(aError) {} 49 MediaData::Type mType; 50 MediaResult mError; 51 }; 52 53 struct MetadataHolder { 54 UniquePtr<MediaInfo> mInfo; 55 UniquePtr<MetadataTags> mTags; 56 }; 57 58 typedef void* MediaDecoderOwnerID; 59 60 struct MOZ_STACK_CLASS MediaFormatReaderInit { 61 MediaResource* mResource = nullptr; 62 VideoFrameContainer* mVideoFrameContainer = nullptr; 63 FrameStatistics* mFrameStats = nullptr; 64 already_AddRefed<layers::KnowsCompositor> mKnowsCompositor; 65 already_AddRefed<GMPCrashHelper> mCrashHelper; 66 // Used in bug 1393399 for temporary telemetry. 67 MediaDecoderOwnerID mMediaDecoderOwnerID = nullptr; 68 }; 69 70 DDLoggedTypeDeclName(MediaFormatReader); 71 72 class MediaFormatReader final 73 : public DecoderDoctorLifeLogger<MediaFormatReader> { 74 static const bool IsExclusive = true; 75 typedef TrackInfo::TrackType TrackType; 76 typedef MozPromise<bool, MediaResult, IsExclusive> NotifyDataArrivedPromise; 77 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaFormatReader) 78 79 public: 80 using TrackSet = EnumSet<TrackInfo::TrackType>; 81 using MetadataPromise = MozPromise<MetadataHolder, MediaResult, IsExclusive>; 82 83 template <typename Type> 84 using DataPromise = MozPromise<RefPtr<Type>, MediaResult, IsExclusive>; 85 using AudioDataPromise = DataPromise<AudioData>; 86 using VideoDataPromise = DataPromise<VideoData>; 87 88 using SeekPromise = MozPromise<media::TimeUnit, SeekRejectValue, IsExclusive>; 89 90 // Note that, conceptually, WaitForData makes sense in a non-exclusive sense. 91 // But in the current architecture it's only ever used exclusively (by MDSM), 92 // so we mark it that way to verify our assumptions. If you have a use-case 93 // for multiple WaitForData consumers, feel free to flip the exclusivity here. 94 using WaitForDataPromise = 95 MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>; 96 97 MediaFormatReader(MediaFormatReaderInit& aInit, MediaDataDemuxer* aDemuxer); 98 99 // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE 100 // on failure. 101 nsresult Init(); 102 103 size_t SizeOfVideoQueueInFrames(); 104 size_t SizeOfAudioQueueInFrames(); 105 106 // Requests one video sample from the reader. 107 RefPtr<VideoDataPromise> RequestVideoData( 108 const media::TimeUnit& aTimeThreshold); 109 110 // Requests one audio sample from the reader. 111 // 112 // The decode should be performed asynchronously, and the promise should 113 // be resolved when it is complete. 114 RefPtr<AudioDataPromise> RequestAudioData(); 115 116 // The default implementation of AsyncReadMetadata is implemented in terms of 117 // synchronous ReadMetadata() calls. Implementations may also 118 // override AsyncReadMetadata to create a more proper async implementation. 119 RefPtr<MetadataPromise> AsyncReadMetadata(); 120 121 // Fills aInfo with the latest cached data required to present the media, 122 // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded. 123 void ReadUpdatedMetadata(MediaInfo* aInfo); 124 125 RefPtr<SeekPromise> Seek(const SeekTarget& aTarget); 126 127 // Called once new data has been cached by the MediaResource. 128 // mBuffered should be recalculated and updated accordingly. 129 void NotifyDataArrived(); 130 131 protected: 132 // Recomputes mBuffered. 133 void UpdateBuffered(); 134 135 public: 136 // Called by MDSM in dormant state to release resources allocated by this 137 // reader. The reader can resume decoding by calling Seek() to a specific 138 // position. 139 void ReleaseResources(); 140 OnTaskQueue()141 bool OnTaskQueue() const { return OwnerThread()->IsCurrentThreadIn(); } 142 143 // Resets all state related to decoding, emptying all buffers etc. 144 // Cancels all pending Request*Data() request callbacks, rejects any 145 // outstanding seek promises, and flushes the decode pipeline. The 146 // decoder must not call any of the callbacks for outstanding 147 // Request*Data() calls after this is called. Calls to Request*Data() 148 // made after this should be processed as usual. 149 // 150 // Normally this call preceedes a Seek() call, or shutdown. 151 // 152 // aParam is a set of TrackInfo::TrackType enums specifying which 153 // queues need to be reset, defaulting to both audio and video tracks. 154 nsresult ResetDecode(TrackSet aTracks); 155 156 // Destroys the decoding state. The reader cannot be made usable again. 157 // This is different from ReleaseMediaResources() as it is irreversable, 158 // whereas ReleaseMediaResources() is. Must be called on the decode 159 // thread. 160 RefPtr<ShutdownPromise> Shutdown(); 161 162 // Returns true if this decoder reader uses hardware accelerated video 163 // decoding. 164 bool VideoIsHardwareAccelerated() const; 165 166 // By default, the state machine polls the reader once per second when it's 167 // in buffering mode. Some readers support a promise-based mechanism by which 168 // they notify the state machine when the data arrives. IsWaitForDataSupported()169 bool IsWaitForDataSupported() const { return true; } 170 171 RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType); 172 173 // The MediaDecoderStateMachine uses various heuristics that assume that 174 // raw media data is arriving sequentially from a network channel. This 175 // makes sense in the <video src="foo"> case, but not for more advanced use 176 // cases like MSE. UseBufferingHeuristics()177 bool UseBufferingHeuristics() const { return mTrackDemuxersMayBlock; } 178 179 RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy); 180 181 // Returns a MediaDebugInfo structure 182 // Used for debugging purposes. 183 void GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo); 184 185 // Switch the video decoder to NullDecoderModule. It might takes effective 186 // since a few samples later depends on how much demuxed samples are already 187 // queued in the original video decoder. 188 void SetVideoNullDecode(bool aIsNullDecode); 189 190 void UpdateCompositor(already_AddRefed<layers::KnowsCompositor>); 191 UpdateDuration(const media::TimeUnit & aDuration)192 void UpdateDuration(const media::TimeUnit& aDuration) { 193 MOZ_ASSERT(OnTaskQueue()); 194 UpdateBuffered(); 195 } 196 CanonicalBuffered()197 AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() { 198 return &mBuffered; 199 } 200 OwnerThread()201 TaskQueue* OwnerThread() const { return mTaskQueue; } 202 TimedMetadataEvent()203 TimedMetadataEventSource& TimedMetadataEvent() { return mTimedMetadataEvent; } 204 205 // Notified by the OggDemuxer during playback when chained ogg is detected. OnMediaNotSeekable()206 MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; } 207 TimedMetadataProducer()208 TimedMetadataEventProducer& TimedMetadataProducer() { 209 return mTimedMetadataEvent; 210 } 211 MediaNotSeekableProducer()212 MediaEventProducer<void>& MediaNotSeekableProducer() { 213 return mOnMediaNotSeekable; 214 } 215 216 // Notified if the reader can't decode a sample due to a missing decryption 217 // key. OnTrackWaitingForKey()218 MediaEventSource<TrackInfo::TrackType>& OnTrackWaitingForKey() { 219 return mOnTrackWaitingForKey; 220 } 221 OnTrackWaitingForKeyProducer()222 MediaEventProducer<TrackInfo::TrackType>& OnTrackWaitingForKeyProducer() { 223 return mOnTrackWaitingForKey; 224 } 225 OnEncrypted()226 MediaEventSource<nsTArray<uint8_t>, nsString>& OnEncrypted() { 227 return mOnEncrypted; 228 } 229 OnWaitingForKey()230 MediaEventSource<void>& OnWaitingForKey() { return mOnWaitingForKey; } 231 OnDecodeWarning()232 MediaEventSource<MediaResult>& OnDecodeWarning() { return mOnDecodeWarning; } 233 OnStoreDecoderBenchmark()234 MediaEventSource<VideoInfo>& OnStoreDecoderBenchmark() { 235 return mOnStoreDecoderBenchmark; 236 } 237 238 private: 239 ~MediaFormatReader(); 240 HasVideo()241 bool HasVideo() const { return mVideo.mTrackDemuxer; } HasAudio()242 bool HasAudio() const { return mAudio.mTrackDemuxer; } 243 244 bool IsWaitingOnCDMResource(); 245 246 bool InitDemuxer(); 247 // Notify the track demuxers that new data has been received. 248 void NotifyTrackDemuxers(); 249 void ReturnOutput(MediaData* aData, TrackType aTrack); 250 251 // Enqueues a task to call Update(aTrack) on the decoder task queue. 252 // Lock for corresponding track must be held. 253 void ScheduleUpdate(TrackType aTrack); 254 void Update(TrackType aTrack); 255 // Handle actions should more data be received. 256 // Returns true if no more action is required. 257 bool UpdateReceivedNewData(TrackType aTrack); 258 // Called when new samples need to be demuxed. 259 void RequestDemuxSamples(TrackType aTrack); 260 // Handle demuxed samples by the input behavior. 261 void HandleDemuxedSamples(TrackType aTrack, 262 FrameStatistics::AutoNotifyDecoded& aA); 263 // Decode any pending already demuxed samples. 264 void DecodeDemuxedSamples(TrackType aTrack, MediaRawData* aSample); 265 266 struct InternalSeekTarget { InternalSeekTargetInternalSeekTarget267 InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget) 268 : mTime(aTime), 269 mDropTarget(aDropTarget), 270 mWaiting(false), 271 mHasSeeked(false) {} 272 TimeInternalSeekTarget273 media::TimeUnit Time() const { return mTime.mStart; } EndTimeInternalSeekTarget274 media::TimeUnit EndTime() const { return mTime.mEnd; } ContainsInternalSeekTarget275 bool Contains(const media::TimeUnit& aTime) const { 276 return mTime.Contains(aTime); 277 } 278 279 media::TimeInterval mTime; 280 bool mDropTarget; 281 bool mWaiting; 282 bool mHasSeeked; 283 }; 284 285 // Perform an internal seek to aTime. If aDropTarget is true then 286 // the first sample past the target will be dropped. 287 void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget); 288 289 // Drain the current decoder. 290 void DrainDecoder(TrackType aTrack); 291 void NotifyNewOutput(TrackType aTrack, 292 MediaDataDecoder::DecodedData&& aResults); 293 void NotifyError(TrackType aTrack, const MediaResult& aError); 294 void NotifyWaitingForData(TrackType aTrack); 295 void NotifyWaitingForKey(TrackType aTrack); 296 void NotifyEndOfStream(TrackType aTrack); 297 298 void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData); 299 300 // Initializes mLayersBackendType if possible. 301 void InitLayersBackendType(); 302 303 void Reset(TrackType aTrack); 304 void DropDecodedSamples(TrackType aTrack); 305 306 bool ShouldSkip(media::TimeUnit aTimeThreshold); 307 308 void SetVideoDecodeThreshold(); 309 310 size_t SizeOfQueue(TrackType aTrack); 311 312 // Fire a new OnStoreDecoderBenchmark event that will create new 313 // storage of the decoder benchmark. 314 // This is called only on TaskQueue. 315 void NotifyDecoderBenchmarkStore(); 316 317 RefPtr<PDMFactory> mPlatform; 318 RefPtr<PDMFactory> mEncryptedPlatform; 319 320 enum class DrainState { 321 None, 322 DrainRequested, 323 Draining, 324 PartialDrainPending, 325 DrainCompleted, 326 DrainAborted, 327 }; 328 329 class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise> { 330 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder) 331 private: 332 ~SharedShutdownPromiseHolder() = default; 333 }; 334 335 struct DecoderData { DecoderDataDecoderData336 DecoderData(MediaFormatReader* aOwner, MediaData::Type aType, 337 uint32_t aNumOfMaxError) 338 : mOwner(aOwner), 339 mType(aType), 340 mMutex("DecoderData"), 341 mDescription("shutdown"), 342 mUpdateScheduled(false), 343 mDemuxEOS(false), 344 mWaitingForData(false), 345 mWaitingForKey(false), 346 mReceivedNewData(false), 347 mFlushing(false), 348 mFlushed(true), 349 mDrainState(DrainState::None), 350 mNumOfConsecutiveError(0), 351 mMaxConsecutiveError(aNumOfMaxError), 352 mFirstFrameTime(Some(media::TimeUnit::Zero())), 353 mNumSamplesInput(0), 354 mNumSamplesOutput(0), 355 mNumSamplesOutputTotal(0), 356 mNumSamplesSkippedTotal(0), 357 mSizeOfQueue(0), 358 mIsHardwareAccelerated(false), 359 mLastStreamSourceID(UINT32_MAX), 360 mIsNullDecode(false), 361 mHardwareDecodingDisabled(false) { 362 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderData", 363 this); 364 } 365 ~DecoderDataDecoderData366 ~DecoderData() { 367 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderData", 368 this); 369 } 370 371 MediaFormatReader* mOwner; 372 // Disambiguate Audio vs Video. 373 MediaData::Type mType; 374 RefPtr<MediaTrackDemuxer> mTrackDemuxer; 375 // TaskQueue on which decoder can choose to decode. 376 // Only non-null up until the decoder is created. 377 RefPtr<TaskQueue> mTaskQueue; 378 379 // Mutex protecting mDescription, mDecoder, mTrackDemuxer and mWorkingInfo 380 // as those can be read outside the TaskQueue. 381 // They are only written on the TaskQueue however, as such mMutex doesn't 382 // need to be held when those members are read on the TaskQueue. 383 Mutex mMutex; 384 // The platform decoder. 385 RefPtr<MediaDataDecoder> mDecoder; 386 nsCString mDescription; 387 void ShutdownDecoder(); 388 389 // Only accessed from reader's task queue. 390 bool mUpdateScheduled; 391 bool mDemuxEOS; 392 bool mWaitingForData; 393 bool mWaitingForKey; 394 bool mReceivedNewData; 395 396 // Pending seek. 397 MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest; 398 399 // Queued demux samples waiting to be decoded. 400 nsTArray<RefPtr<MediaRawData>> mQueuedSamples; 401 MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; 402 // A WaitingPromise is pending if the demuxer is waiting for data or 403 // if the decoder is waiting for a key. 404 MozPromiseHolder<WaitForDataPromise> mWaitingPromise; HasWaitingPromiseDecoderData405 bool HasWaitingPromise() const { 406 MOZ_ASSERT(mOwner->OnTaskQueue()); 407 return !mWaitingPromise.IsEmpty(); 408 } 409 IsWaitingForDataDecoderData410 bool IsWaitingForData() const { 411 MOZ_ASSERT(mOwner->OnTaskQueue()); 412 return mWaitingForData; 413 } 414 IsWaitingForKeyDecoderData415 bool IsWaitingForKey() const { 416 MOZ_ASSERT(mOwner->OnTaskQueue()); 417 return mWaitingForKey && mDecodeRequest.Exists(); 418 } 419 420 // MediaDataDecoder handler's variables. 421 MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest; 422 bool mFlushing; // True if flush is in action. 423 // Set to true if the last operation run on the decoder was a flush. 424 bool mFlushed; 425 RefPtr<SharedShutdownPromiseHolder> mShutdownPromise; 426 427 MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest; 428 DrainState mDrainState; HasPendingDrainDecoderData429 bool HasPendingDrain() const { return mDrainState != DrainState::None; } HasCompletedDrainDecoderData430 bool HasCompletedDrain() const { 431 return mDrainState == DrainState::DrainCompleted || 432 mDrainState == DrainState::DrainAborted; 433 } RequestDrainDecoderData434 void RequestDrain() { 435 MOZ_RELEASE_ASSERT(mDrainState == DrainState::None); 436 mDrainState = DrainState::DrainRequested; 437 } 438 439 uint32_t mNumOfConsecutiveError; 440 uint32_t mMaxConsecutiveError; 441 // Set when we haven't yet decoded the first frame. 442 // Cleared once the first frame has been decoded. 443 // This is used to determine, upon error, if we should try again to decode 444 // the frame, or skip to the next keyframe. 445 Maybe<media::TimeUnit> mFirstFrameTime; 446 447 Maybe<MediaResult> mError; HasFatalErrorDecoderData448 bool HasFatalError() const { 449 if (!mError.isSome()) { 450 return false; 451 } 452 if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) { 453 // Allow decode errors to be non-fatal, but give up 454 // if we have too many, or if warnings should be treated as errors. 455 return mNumOfConsecutiveError > mMaxConsecutiveError || 456 StaticPrefs::media_playback_warnings_as_errors(); 457 } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 458 // If the caller asked for a new decoder we shouldn't treat 459 // it as fatal. 460 return false; 461 } else { 462 // All other error types are fatal 463 return true; 464 } 465 } 466 467 // If set, all decoded samples prior mTimeThreshold will be dropped. 468 // Used for internal seeking when a change of stream is detected or when 469 // encountering data discontinuity. 470 Maybe<InternalSeekTarget> mTimeThreshold; 471 // Time of last decoded sample returned. 472 Maybe<media::TimeInterval> mLastDecodedSampleTime; 473 474 // Decoded samples returned my mDecoder awaiting being returned to 475 // state machine upon request. 476 nsTArray<RefPtr<MediaData>> mOutput; 477 uint64_t mNumSamplesInput; 478 uint64_t mNumSamplesOutput; 479 uint64_t mNumSamplesOutputTotal; 480 uint64_t mNumSamplesSkippedTotal; 481 482 // These get overridden in the templated concrete class. 483 // Indicate if we have a pending promise for decoded frame. 484 // Rejecting the promise will stop the reader from decoding ahead. 485 virtual bool HasPromise() const = 0; 486 virtual void RejectPromise(const MediaResult& aError, 487 const char* aMethodName) = 0; 488 489 // Clear track demuxer related data. ResetDemuxerDecoderData490 void ResetDemuxer() { 491 mDemuxRequest.DisconnectIfExists(); 492 mSeekRequest.DisconnectIfExists(); 493 mTrackDemuxer->Reset(); 494 mQueuedSamples.Clear(); 495 } 496 497 // Flush the decoder if present and reset decoding related data. 498 // Following a flush, the decoder is ready to accept any new data. 499 void Flush(); 500 CancelWaitingForKeyDecoderData501 bool CancelWaitingForKey() { 502 if (!mWaitingForKey) { 503 return false; 504 } 505 mWaitingForKey = false; 506 if (IsWaitingForData() || !HasWaitingPromise()) { 507 return false; 508 } 509 mWaitingPromise.Resolve(mType, __func__); 510 return true; 511 } 512 513 // Reset the state of the DecoderData, clearing all queued frames 514 // (pending demuxed and decoded). 515 // The track demuxer is *not* reset. ResetStateDecoderData516 void ResetState() { 517 MOZ_ASSERT(mOwner->OnTaskQueue()); 518 mDemuxEOS = false; 519 mWaitingForData = false; 520 mQueuedSamples.Clear(); 521 mDecodeRequest.DisconnectIfExists(); 522 mDrainRequest.DisconnectIfExists(); 523 mDrainState = DrainState::None; 524 CancelWaitingForKey(); 525 mTimeThreshold.reset(); 526 mLastDecodedSampleTime.reset(); 527 mOutput.Clear(); 528 mNumSamplesInput = 0; 529 mNumSamplesOutput = 0; 530 mSizeOfQueue = 0; 531 mNextStreamSourceID.reset(); 532 if (!HasFatalError()) { 533 mError.reset(); 534 } 535 } 536 HasInternalSeekPendingDecoderData537 bool HasInternalSeekPending() const { 538 return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked; 539 } 540 541 // Return the current TrackInfo in the stream. If the stream content never 542 // changed since AsyncReadMetadata was called then the TrackInfo used is 543 // mOriginalInfo, other it will be mInfo. The later case is only ever true 544 // with MSE or the WebMDemuxer. GetCurrentInfoDecoderData545 const TrackInfo* GetCurrentInfo() const { 546 if (mInfo) { 547 return *mInfo; 548 } 549 return mOriginalInfo.get(); 550 } 551 // Return the current TrackInfo updated as per the decoder output. 552 // Typically for audio, the number of channels and/or sampling rate can vary 553 // between what was found in the metadata and what the decoder returned. GetWorkingInfoDecoderData554 const TrackInfo* GetWorkingInfo() const { return mWorkingInfo.get(); } IsEncryptedDecoderData555 bool IsEncrypted() const { return GetCurrentInfo()->mCrypto.IsEncrypted(); } 556 557 // Used by the MDSM for logging purposes. 558 Atomic<size_t> mSizeOfQueue; 559 // Used by the MDSM to determine if video decoding is hardware accelerated. 560 // This value is updated after a frame is successfully decoded. 561 Atomic<bool> mIsHardwareAccelerated; 562 // Sample format monitoring. 563 uint32_t mLastStreamSourceID; 564 Maybe<uint32_t> mNextStreamSourceID; 565 media::TimeIntervals mTimeRanges; 566 Maybe<media::TimeUnit> mLastTimeRangesEnd; 567 // TrackInfo as first discovered during ReadMetadata. 568 UniquePtr<TrackInfo> mOriginalInfo; 569 // Written exclusively on the TaskQueue, can be read on MDSM's TaskQueue. 570 // Must be read with parent's mutex held. 571 UniquePtr<TrackInfo> mWorkingInfo; 572 RefPtr<TrackInfoSharedPtr> mInfo; 573 Maybe<media::TimeUnit> mFirstDemuxedSampleTime; 574 // Use NullDecoderModule or not. 575 bool mIsNullDecode; 576 bool mHardwareDecodingDisabled; 577 578 class { 579 public: MeanDecoderData580 float Mean() const { return mMean; } 581 UpdateDecoderData582 void Update(const media::TimeUnit& aValue) { 583 if (aValue == media::TimeUnit::Zero()) { 584 return; 585 } 586 mMean += (1.0f / aValue.ToSeconds() - mMean) / ++mCount; 587 } 588 ResetDecoderData589 void Reset() { 590 mMean = 0; 591 mCount = 0; 592 } 593 594 private: 595 float mMean = 0; 596 uint64_t mCount = 0; 597 } mMeanRate; 598 }; 599 600 template <typename Type> 601 class DecoderDataWithPromise : public DecoderData { 602 public: DecoderDataWithPromise(MediaFormatReader * aOwner,MediaData::Type aType,uint32_t aNumOfMaxError)603 DecoderDataWithPromise(MediaFormatReader* aOwner, MediaData::Type aType, 604 uint32_t aNumOfMaxError) 605 : DecoderData(aOwner, aType, aNumOfMaxError), mHasPromise(false) { 606 DecoderDoctorLogger::LogConstructionAndBase( 607 "MediaFormatReader::DecoderDataWithPromise", this, 608 "MediaFormatReader::DecoderData", 609 static_cast<const MediaFormatReader::DecoderData*>(this)); 610 } 611 ~DecoderDataWithPromise()612 ~DecoderDataWithPromise() { 613 DecoderDoctorLogger::LogDestruction( 614 "MediaFormatReader::DecoderDataWithPromise", this); 615 } 616 HasPromise()617 bool HasPromise() const override { return mHasPromise; } 618 EnsurePromise(const char * aMethodName)619 RefPtr<DataPromise<Type>> EnsurePromise(const char* aMethodName) { 620 MOZ_ASSERT(mOwner->OnTaskQueue()); 621 mHasPromise = true; 622 return mPromise.Ensure(aMethodName); 623 } 624 ResolvePromise(Type * aData,const char * aMethodName)625 void ResolvePromise(Type* aData, const char* aMethodName) { 626 MOZ_ASSERT(mOwner->OnTaskQueue()); 627 mPromise.Resolve(aData, aMethodName); 628 mHasPromise = false; 629 } 630 RejectPromise(const MediaResult & aError,const char * aMethodName)631 void RejectPromise(const MediaResult& aError, 632 const char* aMethodName) override { 633 MOZ_ASSERT(mOwner->OnTaskQueue()); 634 mPromise.Reject(aError, aMethodName); 635 mHasPromise = false; 636 } 637 638 private: 639 MozPromiseHolder<DataPromise<Type>> mPromise; 640 Atomic<bool> mHasPromise; 641 }; 642 643 // Decode task queue. 644 RefPtr<TaskQueue> mTaskQueue; 645 646 DecoderDataWithPromise<AudioData> mAudio; 647 DecoderDataWithPromise<VideoData> mVideo; 648 649 // Returns true when the decoder for this track needs input. 650 bool NeedInput(DecoderData& aDecoder); 651 652 DecoderData& GetDecoderData(TrackType aTrack); 653 654 // Demuxer objects. 655 class DemuxerProxy; 656 UniquePtr<DemuxerProxy> mDemuxer; 657 bool mDemuxerInitDone; 658 void OnDemuxerInitDone(const MediaResult& aResult); 659 void OnDemuxerInitFailed(const MediaResult& aError); 660 MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; 661 MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise; 662 bool mPendingNotifyDataArrived; 663 void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); 664 665 void DoDemuxVideo(); 666 void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnVideoDemuxFailed(const MediaResult & aError)667 void OnVideoDemuxFailed(const MediaResult& aError) { 668 OnDemuxFailed(TrackType::kVideoTrack, aError); 669 } 670 671 void DoDemuxAudio(); 672 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnAudioDemuxFailed(const MediaResult & aError)673 void OnAudioDemuxFailed(const MediaResult& aError) { 674 OnDemuxFailed(TrackType::kAudioTrack, aError); 675 } 676 677 void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold); 678 MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> 679 mSkipRequest; 680 void VideoSkipReset(uint32_t aSkipped); 681 void OnVideoSkipCompleted(uint32_t aSkipped); 682 void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure); 683 684 // The last number of decoded output frames that we've reported to 685 // MediaDecoder::NotifyDecoded(). We diff the number of output video 686 // frames every time that DecodeVideoData() is called, and report the 687 // delta there. 688 uint64_t mLastReportedNumDecodedFrames; 689 690 // Timestamp of the previous decoded keyframe, in microseconds. 691 int64_t mPreviousDecodedKeyframeTime_us; 692 // Default mLastDecodedKeyframeTime_us value, must be bigger than anything. 693 static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX; 694 695 RefPtr<layers::KnowsCompositor> mKnowsCompositor; 696 697 // Metadata objects 698 // True if we've read the streams' metadata. 699 bool mInitDone; 700 MozPromiseHolder<MetadataPromise> mMetadataPromise; 701 bool IsEncrypted() const; 702 703 // Set to true if any of our track buffers may be blocking. 704 bool mTrackDemuxersMayBlock; 705 706 // Seeking objects. 707 void SetSeekTarget(const SeekTarget& aTarget); IsSeeking()708 bool IsSeeking() const { return mPendingSeekTime.isSome(); } IsVideoSeeking()709 bool IsVideoSeeking() const { 710 return IsSeeking() && mOriginalSeekTarget.IsVideoOnly(); 711 } 712 void ScheduleSeek(); 713 void AttemptSeek(); 714 void OnSeekFailed(TrackType aTrack, const MediaResult& aError); 715 void DoVideoSeek(); 716 void OnVideoSeekCompleted(media::TimeUnit aTime); 717 void OnVideoSeekFailed(const MediaResult& aError); 718 bool mSeekScheduled; 719 720 void DoAudioSeek(); 721 void OnAudioSeekCompleted(media::TimeUnit aTime); 722 void OnAudioSeekFailed(const MediaResult& aError); 723 // The SeekTarget that was last given to Seek() 724 SeekTarget mOriginalSeekTarget; 725 // Temporary seek information while we wait for the data 726 Maybe<media::TimeUnit> mFallbackSeekTime; 727 Maybe<media::TimeUnit> mPendingSeekTime; 728 MozPromiseHolder<SeekPromise> mSeekPromise; 729 730 RefPtr<VideoFrameContainer> mVideoFrameContainer; 731 layers::ImageContainer* GetImageContainer(); 732 733 RefPtr<CDMProxy> mCDMProxy; 734 735 RefPtr<GMPCrashHelper> mCrashHelper; 736 737 void SetNullDecode(TrackType aTrack, bool aIsNullDecode); 738 739 class DecoderFactory; 740 UniquePtr<DecoderFactory> mDecoderFactory; 741 742 class ShutdownPromisePool; 743 UniquePtr<ShutdownPromisePool> mShutdownPromisePool; 744 745 MediaEventListener mOnTrackWaitingForKeyListener; 746 747 void OnFirstDemuxCompleted(TrackInfo::TrackType aType, 748 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); 749 750 void OnFirstDemuxFailed(TrackInfo::TrackType aType, 751 const MediaResult& aError); 752 753 void MaybeResolveMetadataPromise(); 754 755 // Stores presentation info required for playback. 756 MediaInfo mInfo; 757 758 UniquePtr<MetadataTags> mTags; 759 760 // A flag indicating if the start time is known or not. 761 bool mHasStartTime = false; 762 763 void ShutdownDecoder(TrackType aTrack); 764 RefPtr<ShutdownPromise> TearDownDecoders(); 765 766 bool mShutdown = false; 767 768 // Buffered range. 769 Canonical<media::TimeIntervals> mBuffered; 770 771 // Used to send TimedMetadata to the listener. 772 TimedMetadataEventProducer mTimedMetadataEvent; 773 774 // Notify if this media is not seekable. 775 MediaEventProducer<void> mOnMediaNotSeekable; 776 777 // Notify if we are waiting for a decryption key. 778 MediaEventProducer<TrackInfo::TrackType> mOnTrackWaitingForKey; 779 780 MediaEventProducer<nsTArray<uint8_t>, nsString> mOnEncrypted; 781 782 MediaEventProducer<void> mOnWaitingForKey; 783 784 MediaEventProducer<MediaResult> mOnDecodeWarning; 785 786 MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark; 787 788 RefPtr<FrameStatistics> mFrameStats; 789 790 // Used in bug 1393399 for telemetry. 791 const MediaDecoderOwnerID mMediaDecoderOwnerID; 792 793 bool ResolveSetCDMPromiseIfDone(TrackType aTrack); 794 void PrepareToSetCDMForTrack(TrackType aTrack); 795 MozPromiseHolder<SetCDMPromise> mSetCDMPromise; 796 TrackSet mSetCDMForTracks{}; 797 bool IsDecoderWaitingForCDM(TrackType aTrack); 798 }; 799 800 } // namespace mozilla 801 802 #endif 803