1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #if !defined(MediaFormatReader_h_) 8 # define MediaFormatReader_h_ 9 10 # include "FrameStatistics.h" 11 # include "MediaDataDemuxer.h" 12 # include "MediaEventSource.h" 13 # include "MediaMetadataManager.h" 14 # include "MediaPromiseDefs.h" 15 # include "PlatformDecoderModule.h" 16 # include "SeekTarget.h" 17 # include "mozilla/Atomics.h" 18 # include "mozilla/Maybe.h" 19 # include "mozilla/Mutex.h" 20 # include "mozilla/StateMirroring.h" 21 # include "mozilla/StaticPrefs_media.h" 22 # include "mozilla/TaskQueue.h" 23 # include "mozilla/ThreadSafeWeakPtr.h" 24 # include "mozilla/dom/MediaDebugInfoBinding.h" 25 26 namespace mozilla { 27 28 class CDMProxy; 29 class GMPCrashHelper; 30 class MediaResource; 31 class VideoFrameContainer; 32 33 struct WaitForDataRejectValue { 34 enum Reason { SHUTDOWN, CANCELED }; 35 WaitForDataRejectValueWaitForDataRejectValue36 WaitForDataRejectValue(MediaData::Type aType, Reason aReason) 37 : mType(aType), mReason(aReason) {} 38 MediaData::Type mType; 39 Reason mReason; 40 }; 41 42 struct SeekRejectValue { SeekRejectValueSeekRejectValue43 MOZ_IMPLICIT SeekRejectValue(const MediaResult& aError) 44 : mType(MediaData::Type::NULL_DATA), mError(aError) {} SeekRejectValueSeekRejectValue45 MOZ_IMPLICIT SeekRejectValue(nsresult aResult) 46 : mType(MediaData::Type::NULL_DATA), mError(aResult) {} SeekRejectValueSeekRejectValue47 SeekRejectValue(MediaData::Type aType, const MediaResult& aError) 48 : mType(aType), mError(aError) {} 49 MediaData::Type mType; 50 MediaResult mError; 51 }; 52 53 struct MetadataHolder { 54 UniquePtr<MediaInfo> mInfo; 55 UniquePtr<MetadataTags> mTags; 56 }; 57 58 typedef void* MediaDecoderOwnerID; 59 60 struct MOZ_STACK_CLASS MediaFormatReaderInit { 61 MediaResource* mResource = nullptr; 62 VideoFrameContainer* mVideoFrameContainer = nullptr; 63 FrameStatistics* mFrameStats = nullptr; 64 already_AddRefed<layers::KnowsCompositor> mKnowsCompositor; 65 already_AddRefed<GMPCrashHelper> mCrashHelper; 66 // Used in bug 1393399 for temporary telemetry. 67 MediaDecoderOwnerID mMediaDecoderOwnerID = nullptr; 68 }; 69 70 DDLoggedTypeDeclName(MediaFormatReader); 71 72 class MediaFormatReader final 73 : public SupportsThreadSafeWeakPtr<MediaFormatReader>, 74 public DecoderDoctorLifeLogger<MediaFormatReader> { 75 static const bool IsExclusive = true; 76 typedef TrackInfo::TrackType TrackType; 77 typedef MozPromise<bool, MediaResult, IsExclusive> NotifyDataArrivedPromise; 78 79 public: 80 MOZ_DECLARE_THREADSAFEWEAKREFERENCE_TYPENAME(MediaFormatReader) 81 MOZ_DECLARE_REFCOUNTED_TYPENAME(MediaFormatReader) 82 83 using TrackSet = EnumSet<TrackInfo::TrackType>; 84 using MetadataPromise = MozPromise<MetadataHolder, MediaResult, IsExclusive>; 85 86 template <typename Type> 87 using DataPromise = MozPromise<RefPtr<Type>, MediaResult, IsExclusive>; 88 using AudioDataPromise = DataPromise<AudioData>; 89 using VideoDataPromise = DataPromise<VideoData>; 90 91 using SeekPromise = MozPromise<media::TimeUnit, SeekRejectValue, IsExclusive>; 92 93 // Note that, conceptually, WaitForData makes sense in a non-exclusive sense. 94 // But in the current architecture it's only ever used exclusively (by MDSM), 95 // so we mark it that way to verify our assumptions. If you have a use-case 96 // for multiple WaitForData consumers, feel free to flip the exclusivity here. 97 using WaitForDataPromise = 98 MozPromise<MediaData::Type, WaitForDataRejectValue, IsExclusive>; 99 100 MediaFormatReader(MediaFormatReaderInit& aInit, MediaDataDemuxer* aDemuxer); 101 virtual ~MediaFormatReader(); 102 103 // Initializes the reader, returns NS_OK on success, or NS_ERROR_FAILURE 104 // on failure. 105 nsresult Init(); 106 107 size_t SizeOfVideoQueueInFrames(); 108 size_t SizeOfAudioQueueInFrames(); 109 110 // Requests one video sample from the reader. 111 RefPtr<VideoDataPromise> RequestVideoData( 112 const media::TimeUnit& aTimeThreshold, 113 bool aRequestNextVideoKeyFrame = false); 114 115 // Requests one audio sample from the reader. 116 // 117 // The decode should be performed asynchronously, and the promise should 118 // be resolved when it is complete. 119 RefPtr<AudioDataPromise> RequestAudioData(); 120 121 // The default implementation of AsyncReadMetadata is implemented in terms of 122 // synchronous ReadMetadata() calls. Implementations may also 123 // override AsyncReadMetadata to create a more proper async implementation. 124 RefPtr<MetadataPromise> AsyncReadMetadata(); 125 126 // Fills aInfo with the latest cached data required to present the media, 127 // ReadUpdatedMetadata will always be called once ReadMetadata has succeeded. 128 void ReadUpdatedMetadata(MediaInfo* aInfo); 129 130 RefPtr<SeekPromise> Seek(const SeekTarget& aTarget); 131 132 // Called once new data has been cached by the MediaResource. 133 // mBuffered should be recalculated and updated accordingly. 134 void NotifyDataArrived(); 135 136 protected: 137 // Recomputes mBuffered. 138 void UpdateBuffered(); 139 140 public: 141 // Called by MDSM in dormant state to release resources allocated by this 142 // reader. The reader can resume decoding by calling Seek() to a specific 143 // position. 144 void ReleaseResources(); 145 OnTaskQueue()146 bool OnTaskQueue() const { return OwnerThread()->IsCurrentThreadIn(); } 147 148 // Resets all state related to decoding, emptying all buffers etc. 149 // Cancels all pending Request*Data() request callbacks, rejects any 150 // outstanding seek promises, and flushes the decode pipeline. The 151 // decoder must not call any of the callbacks for outstanding 152 // Request*Data() calls after this is called. Calls to Request*Data() 153 // made after this should be processed as usual. 154 // 155 // Normally this call preceedes a Seek() call, or shutdown. 156 // 157 // aParam is a set of TrackInfo::TrackType enums specifying which 158 // queues need to be reset, defaulting to both audio and video tracks. 159 nsresult ResetDecode(TrackSet aTracks); 160 161 // Destroys the decoding state. The reader cannot be made usable again. 162 // This is different from ReleaseMediaResources() as it is irreversable, 163 // whereas ReleaseMediaResources() is. Must be called on the decode 164 // thread. 165 RefPtr<ShutdownPromise> Shutdown(); 166 167 // Returns true if this decoder reader uses hardware accelerated video 168 // decoding. 169 bool VideoIsHardwareAccelerated() const; 170 171 // By default, the state machine polls the reader once per second when it's 172 // in buffering mode. Some readers support a promise-based mechanism by which 173 // they notify the state machine when the data arrives. IsWaitForDataSupported()174 bool IsWaitForDataSupported() const { return true; } 175 176 RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType); 177 178 // The MediaDecoderStateMachine uses various heuristics that assume that 179 // raw media data is arriving sequentially from a network channel. This 180 // makes sense in the <video src="foo"> case, but not for more advanced use 181 // cases like MSE. UseBufferingHeuristics()182 bool UseBufferingHeuristics() const { return mTrackDemuxersMayBlock; } 183 184 RefPtr<SetCDMPromise> SetCDMProxy(CDMProxy* aProxy); 185 186 // Returns a MediaDebugInfo structure 187 // Used for debugging purposes. 188 void GetDebugInfo(dom::MediaFormatReaderDebugInfo& aInfo); 189 190 // Switch the video decoder to NullDecoderModule. It might takes effective 191 // since a few samples later depends on how much demuxed samples are already 192 // queued in the original video decoder. 193 void SetVideoNullDecode(bool aIsNullDecode); 194 195 void UpdateCompositor(already_AddRefed<layers::KnowsCompositor>); 196 UpdateDuration(const media::TimeUnit & aDuration)197 void UpdateDuration(const media::TimeUnit& aDuration) { 198 MOZ_ASSERT(OnTaskQueue()); 199 UpdateBuffered(); 200 } 201 CanonicalBuffered()202 AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() { 203 return &mBuffered; 204 } 205 OwnerThread()206 TaskQueue* OwnerThread() const { return mTaskQueue; } 207 TimedMetadataEvent()208 TimedMetadataEventSource& TimedMetadataEvent() { return mTimedMetadataEvent; } 209 210 // Notified by the OggDemuxer during playback when chained ogg is detected. OnMediaNotSeekable()211 MediaEventSource<void>& OnMediaNotSeekable() { return mOnMediaNotSeekable; } 212 TimedMetadataProducer()213 TimedMetadataEventProducer& TimedMetadataProducer() { 214 return mTimedMetadataEvent; 215 } 216 MediaNotSeekableProducer()217 MediaEventProducer<void>& MediaNotSeekableProducer() { 218 return mOnMediaNotSeekable; 219 } 220 221 // Notified if the reader can't decode a sample due to a missing decryption 222 // key. OnTrackWaitingForKey()223 MediaEventSource<TrackInfo::TrackType>& OnTrackWaitingForKey() { 224 return mOnTrackWaitingForKey; 225 } 226 OnTrackWaitingForKeyProducer()227 MediaEventProducer<TrackInfo::TrackType>& OnTrackWaitingForKeyProducer() { 228 return mOnTrackWaitingForKey; 229 } 230 OnEncrypted()231 MediaEventSource<nsTArray<uint8_t>, nsString>& OnEncrypted() { 232 return mOnEncrypted; 233 } 234 OnWaitingForKey()235 MediaEventSource<void>& OnWaitingForKey() { return mOnWaitingForKey; } 236 OnDecodeWarning()237 MediaEventSource<MediaResult>& OnDecodeWarning() { return mOnDecodeWarning; } 238 OnStoreDecoderBenchmark()239 MediaEventSource<VideoInfo>& OnStoreDecoderBenchmark() { 240 return mOnStoreDecoderBenchmark; 241 } 242 243 private: HasVideo()244 bool HasVideo() const { return mVideo.mTrackDemuxer; } HasAudio()245 bool HasAudio() const { return mAudio.mTrackDemuxer; } 246 247 bool IsWaitingOnCDMResource(); 248 249 bool InitDemuxer(); 250 // Notify the track demuxers that new data has been received. 251 void NotifyTrackDemuxers(); 252 void ReturnOutput(MediaData* aData, TrackType aTrack); 253 254 // Enqueues a task to call Update(aTrack) on the decoder task queue. 255 // Lock for corresponding track must be held. 256 void ScheduleUpdate(TrackType aTrack); 257 void Update(TrackType aTrack); 258 // Handle actions should more data be received. 259 // Returns true if no more action is required. 260 bool UpdateReceivedNewData(TrackType aTrack); 261 // Called when new samples need to be demuxed. 262 void RequestDemuxSamples(TrackType aTrack); 263 // Handle demuxed samples by the input behavior. 264 void HandleDemuxedSamples(TrackType aTrack, 265 FrameStatistics::AutoNotifyDecoded& aA); 266 // Decode any pending already demuxed samples. 267 void DecodeDemuxedSamples(TrackType aTrack, MediaRawData* aSample); 268 269 struct InternalSeekTarget { InternalSeekTargetInternalSeekTarget270 InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget) 271 : mTime(aTime), 272 mDropTarget(aDropTarget), 273 mWaiting(false), 274 mHasSeeked(false) {} 275 TimeInternalSeekTarget276 media::TimeUnit Time() const { return mTime.mStart; } EndTimeInternalSeekTarget277 media::TimeUnit EndTime() const { return mTime.mEnd; } ContainsInternalSeekTarget278 bool Contains(const media::TimeUnit& aTime) const { 279 return mTime.Contains(aTime); 280 } 281 282 media::TimeInterval mTime; 283 bool mDropTarget; 284 bool mWaiting; 285 bool mHasSeeked; 286 }; 287 288 // Perform an internal seek to aTime. If aDropTarget is true then 289 // the first sample past the target will be dropped. 290 void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget); 291 // Return the end time of the internal seek target if it exists. Otherwise, 292 // return infinity. 293 media::TimeUnit GetInternalSeekTargetEndTime() const; 294 295 // Drain the current decoder. 296 void DrainDecoder(TrackType aTrack); 297 void NotifyNewOutput(TrackType aTrack, 298 MediaDataDecoder::DecodedData&& aResults); 299 void NotifyError(TrackType aTrack, const MediaResult& aError); 300 void NotifyWaitingForData(TrackType aTrack); 301 void NotifyWaitingForKey(TrackType aTrack); 302 void NotifyEndOfStream(TrackType aTrack); 303 304 void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData); 305 306 // Initializes mLayersBackendType if possible. 307 void InitLayersBackendType(); 308 309 void Reset(TrackType aTrack); 310 void DropDecodedSamples(TrackType aTrack); 311 312 // Return a target timeunit which the reader should skip to, this would be 313 // either the timethreshold we pass, or the time of the next keyframe. Return 314 // nothing if we don't need to skip. 315 // @param aTimeThreshold 316 // The time that we expect the time of next video frame should be or go beyond 317 // @param aRequestNextVideoKeyFrame 318 // If true and the next keyframe's time is larger than aTimeThreshold, skip to 319 // the next keyframe time instead of aTimeThreshold. 320 Maybe<media::TimeUnit> ShouldSkip(media::TimeUnit aTimeThreshold, 321 bool aRequestNextVideoKeyFrame); 322 323 void SetVideoDecodeThreshold(); 324 325 size_t SizeOfQueue(TrackType aTrack); 326 327 // Fire a new OnStoreDecoderBenchmark event that will create new 328 // storage of the decoder benchmark. 329 // This is called only on TaskQueue. 330 void NotifyDecoderBenchmarkStore(); 331 332 enum class DrainState { 333 None, 334 DrainRequested, 335 Draining, 336 PartialDrainPending, 337 DrainCompleted, 338 DrainAborted, 339 }; 340 341 class SharedShutdownPromiseHolder : public MozPromiseHolder<ShutdownPromise> { 342 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedShutdownPromiseHolder) 343 private: 344 ~SharedShutdownPromiseHolder() = default; 345 }; 346 347 struct DecoderData { DecoderDataDecoderData348 DecoderData(MediaFormatReader* aOwner, MediaData::Type aType, 349 uint32_t aNumOfMaxError) 350 : mOwner(aOwner), 351 mType(aType), 352 mMutex("DecoderData"), 353 mDescription("shutdown"), 354 mUpdateScheduled(false), 355 mDemuxEOS(false), 356 mWaitingForData(false), 357 mWaitingForKey(false), 358 mReceivedNewData(false), 359 mFlushing(false), 360 mFlushed(true), 361 mDrainState(DrainState::None), 362 mNumOfConsecutiveError(0), 363 mMaxConsecutiveError(aNumOfMaxError), 364 mFirstFrameTime(Some(media::TimeUnit::Zero())), 365 mNumSamplesInput(0), 366 mNumSamplesOutput(0), 367 mNumSamplesOutputTotal(0), 368 mNumSamplesSkippedTotal(0), 369 mSizeOfQueue(0), 370 mIsHardwareAccelerated(false), 371 mLastStreamSourceID(UINT32_MAX), 372 mIsNullDecode(false), 373 mHardwareDecodingDisabled(false) { 374 DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderData", 375 this); 376 } 377 ~DecoderDataDecoderData378 ~DecoderData() { 379 DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderData", 380 this); 381 } 382 383 MediaFormatReader* mOwner; 384 // Disambiguate Audio vs Video. 385 MediaData::Type mType; 386 RefPtr<MediaTrackDemuxer> mTrackDemuxer; 387 // TaskQueue on which decoder can choose to decode. 388 // Only non-null up until the decoder is created. 389 RefPtr<TaskQueue> mTaskQueue; 390 391 // Mutex protecting mDescription, mDecoder, mTrackDemuxer and mWorkingInfo 392 // as those can be read outside the TaskQueue. 393 // They are only written on the TaskQueue however, as such mMutex doesn't 394 // need to be held when those members are read on the TaskQueue. 395 Mutex mMutex; 396 // The platform decoder. 397 RefPtr<MediaDataDecoder> mDecoder; 398 nsCString mDescription; 399 void ShutdownDecoder(); 400 401 // Only accessed from reader's task queue. 402 bool mUpdateScheduled; 403 bool mDemuxEOS; 404 bool mWaitingForData; 405 bool mWaitingForKey; 406 bool mReceivedNewData; 407 408 // Pending seek. 409 MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest; 410 411 // Queued demux samples waiting to be decoded. 412 nsTArray<RefPtr<MediaRawData>> mQueuedSamples; 413 MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; 414 // A WaitingPromise is pending if the demuxer is waiting for data or 415 // if the decoder is waiting for a key. 416 MozPromiseHolder<WaitForDataPromise> mWaitingPromise; HasWaitingPromiseDecoderData417 bool HasWaitingPromise() const { 418 MOZ_ASSERT(mOwner->OnTaskQueue()); 419 return !mWaitingPromise.IsEmpty(); 420 } 421 IsWaitingForDataDecoderData422 bool IsWaitingForData() const { 423 MOZ_ASSERT(mOwner->OnTaskQueue()); 424 return mWaitingForData; 425 } 426 IsWaitingForKeyDecoderData427 bool IsWaitingForKey() const { 428 MOZ_ASSERT(mOwner->OnTaskQueue()); 429 return mWaitingForKey && mDecodeRequest.Exists(); 430 } 431 432 // MediaDataDecoder handler's variables. 433 MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDecodeRequest; 434 bool mFlushing; // True if flush is in action. 435 // Set to true if the last operation run on the decoder was a flush. 436 bool mFlushed; 437 RefPtr<SharedShutdownPromiseHolder> mShutdownPromise; 438 439 MozPromiseRequestHolder<MediaDataDecoder::DecodePromise> mDrainRequest; 440 DrainState mDrainState; HasPendingDrainDecoderData441 bool HasPendingDrain() const { return mDrainState != DrainState::None; } HasCompletedDrainDecoderData442 bool HasCompletedDrain() const { 443 return mDrainState == DrainState::DrainCompleted || 444 mDrainState == DrainState::DrainAborted; 445 } RequestDrainDecoderData446 void RequestDrain() { 447 MOZ_RELEASE_ASSERT(mDrainState == DrainState::None); 448 mDrainState = DrainState::DrainRequested; 449 } 450 451 uint32_t mNumOfConsecutiveError; 452 uint32_t mMaxConsecutiveError; 453 // Set when we haven't yet decoded the first frame. 454 // Cleared once the first frame has been decoded. 455 // This is used to determine, upon error, if we should try again to decode 456 // the frame, or skip to the next keyframe. 457 Maybe<media::TimeUnit> mFirstFrameTime; 458 459 Maybe<MediaResult> mError; HasFatalErrorDecoderData460 bool HasFatalError() const { 461 if (!mError.isSome()) { 462 return false; 463 } 464 if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) { 465 // Allow decode errors to be non-fatal, but give up 466 // if we have too many, or if warnings should be treated as errors. 467 return mNumOfConsecutiveError > mMaxConsecutiveError || 468 StaticPrefs::media_playback_warnings_as_errors(); 469 } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 470 // If the caller asked for a new decoder we shouldn't treat 471 // it as fatal. 472 return false; 473 } else if (mError.ref() == 474 NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_ERR) { 475 // Allow to restart remote decoder. 476 return false; 477 } else { 478 // All other error types are fatal 479 return true; 480 } 481 } 482 483 // If set, all decoded samples prior mTimeThreshold will be dropped. 484 // Used for internal seeking when a change of stream is detected or when 485 // encountering data discontinuity. 486 Maybe<InternalSeekTarget> mTimeThreshold; 487 // Time of last decoded sample returned. 488 Maybe<media::TimeInterval> mLastDecodedSampleTime; 489 490 // Decoded samples returned my mDecoder awaiting being returned to 491 // state machine upon request. 492 nsTArray<RefPtr<MediaData>> mOutput; 493 uint64_t mNumSamplesInput; 494 uint64_t mNumSamplesOutput; 495 uint64_t mNumSamplesOutputTotal; 496 uint64_t mNumSamplesSkippedTotal; 497 498 // These get overridden in the templated concrete class. 499 // Indicate if we have a pending promise for decoded frame. 500 // Rejecting the promise will stop the reader from decoding ahead. 501 virtual bool HasPromise() const = 0; 502 virtual void RejectPromise(const MediaResult& aError, 503 const char* aMethodName) = 0; 504 505 // Clear track demuxer related data. ResetDemuxerDecoderData506 void ResetDemuxer() { 507 mDemuxRequest.DisconnectIfExists(); 508 mSeekRequest.DisconnectIfExists(); 509 mTrackDemuxer->Reset(); 510 mQueuedSamples.Clear(); 511 } 512 513 // Flush the decoder if present and reset decoding related data. 514 // Following a flush, the decoder is ready to accept any new data. 515 void Flush(); 516 CancelWaitingForKeyDecoderData517 bool CancelWaitingForKey() { 518 if (!mWaitingForKey) { 519 return false; 520 } 521 mWaitingForKey = false; 522 if (IsWaitingForData() || !HasWaitingPromise()) { 523 return false; 524 } 525 mWaitingPromise.Resolve(mType, __func__); 526 return true; 527 } 528 529 // Reset the state of the DecoderData, clearing all queued frames 530 // (pending demuxed and decoded). 531 // The track demuxer is *not* reset. ResetStateDecoderData532 void ResetState() { 533 MOZ_ASSERT(mOwner->OnTaskQueue()); 534 mDemuxEOS = false; 535 mWaitingForData = false; 536 mQueuedSamples.Clear(); 537 mDecodeRequest.DisconnectIfExists(); 538 mDrainRequest.DisconnectIfExists(); 539 mDrainState = DrainState::None; 540 CancelWaitingForKey(); 541 mTimeThreshold.reset(); 542 mLastDecodedSampleTime.reset(); 543 mOutput.Clear(); 544 mNumSamplesInput = 0; 545 mNumSamplesOutput = 0; 546 mSizeOfQueue = 0; 547 mNextStreamSourceID.reset(); 548 if (!HasFatalError()) { 549 mError.reset(); 550 } 551 } 552 HasInternalSeekPendingDecoderData553 bool HasInternalSeekPending() const { 554 return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked; 555 } 556 557 // Return the current TrackInfo in the stream. If the stream content never 558 // changed since AsyncReadMetadata was called then the TrackInfo used is 559 // mOriginalInfo, other it will be mInfo. The later case is only ever true 560 // with MSE or the WebMDemuxer. GetCurrentInfoDecoderData561 const TrackInfo* GetCurrentInfo() const { 562 if (mInfo) { 563 return *mInfo; 564 } 565 return mOriginalInfo.get(); 566 } 567 // Return the current TrackInfo updated as per the decoder output. 568 // Typically for audio, the number of channels and/or sampling rate can vary 569 // between what was found in the metadata and what the decoder returned. GetWorkingInfoDecoderData570 const TrackInfo* GetWorkingInfo() const { return mWorkingInfo.get(); } IsEncryptedDecoderData571 bool IsEncrypted() const { return GetCurrentInfo()->mCrypto.IsEncrypted(); } 572 573 // Used by the MDSM for logging purposes. 574 Atomic<size_t> mSizeOfQueue; 575 // Used by the MDSM to determine if video decoding is hardware accelerated. 576 // This value is updated after a frame is successfully decoded. 577 Atomic<bool> mIsHardwareAccelerated; 578 // Sample format monitoring. 579 uint32_t mLastStreamSourceID; 580 Maybe<uint32_t> mNextStreamSourceID; 581 media::TimeIntervals mTimeRanges; 582 Maybe<media::TimeUnit> mLastTimeRangesEnd; 583 // TrackInfo as first discovered during ReadMetadata. 584 UniquePtr<TrackInfo> mOriginalInfo; 585 // Written exclusively on the TaskQueue, can be read on MDSM's TaskQueue. 586 // Must be read with parent's mutex held. 587 UniquePtr<TrackInfo> mWorkingInfo; 588 RefPtr<TrackInfoSharedPtr> mInfo; 589 Maybe<media::TimeUnit> mFirstDemuxedSampleTime; 590 // Use NullDecoderModule or not. 591 bool mIsNullDecode; 592 bool mHardwareDecodingDisabled; 593 // Whether we have reported hardware decoding support for video. Used only 594 // on reader's task queue, 595 bool mHasReportedVideoHardwareSupportTelemtry = false; 596 597 class { 598 public: MeanDecoderData599 float Mean() const { return mMean; } 600 UpdateDecoderData601 void Update(const media::TimeUnit& aValue) { 602 if (aValue == media::TimeUnit::Zero()) { 603 return; 604 } 605 mMean += (1.0f / aValue.ToSeconds() - mMean) / ++mCount; 606 } 607 ResetDecoderData608 void Reset() { 609 mMean = 0; 610 mCount = 0; 611 } 612 613 private: 614 float mMean = 0; 615 uint64_t mCount = 0; 616 } mMeanRate; 617 }; 618 619 template <typename Type> 620 class DecoderDataWithPromise : public DecoderData { 621 public: DecoderDataWithPromise(MediaFormatReader * aOwner,MediaData::Type aType,uint32_t aNumOfMaxError)622 DecoderDataWithPromise(MediaFormatReader* aOwner, MediaData::Type aType, 623 uint32_t aNumOfMaxError) 624 : DecoderData(aOwner, aType, aNumOfMaxError), mHasPromise(false) { 625 DecoderDoctorLogger::LogConstructionAndBase( 626 "MediaFormatReader::DecoderDataWithPromise", this, 627 "MediaFormatReader::DecoderData", 628 static_cast<const MediaFormatReader::DecoderData*>(this)); 629 } 630 ~DecoderDataWithPromise()631 ~DecoderDataWithPromise() { 632 DecoderDoctorLogger::LogDestruction( 633 "MediaFormatReader::DecoderDataWithPromise", this); 634 } 635 HasPromise()636 bool HasPromise() const override { return mHasPromise; } 637 EnsurePromise(const char * aMethodName)638 RefPtr<DataPromise<Type>> EnsurePromise(const char* aMethodName) { 639 MOZ_ASSERT(mOwner->OnTaskQueue()); 640 mHasPromise = true; 641 return mPromise.Ensure(aMethodName); 642 } 643 ResolvePromise(Type * aData,const char * aMethodName)644 void ResolvePromise(Type* aData, const char* aMethodName) { 645 MOZ_ASSERT(mOwner->OnTaskQueue()); 646 mPromise.Resolve(aData, aMethodName); 647 mHasPromise = false; 648 } 649 RejectPromise(const MediaResult & aError,const char * aMethodName)650 void RejectPromise(const MediaResult& aError, 651 const char* aMethodName) override { 652 MOZ_ASSERT(mOwner->OnTaskQueue()); 653 mPromise.Reject(aError, aMethodName); 654 mHasPromise = false; 655 } 656 657 private: 658 MozPromiseHolder<DataPromise<Type>> mPromise; 659 Atomic<bool> mHasPromise; 660 }; 661 662 // Decode task queue. 663 RefPtr<TaskQueue> mTaskQueue; 664 665 DecoderDataWithPromise<AudioData> mAudio; 666 DecoderDataWithPromise<VideoData> mVideo; 667 668 // Returns true when the decoder for this track needs input. 669 bool NeedInput(DecoderData& aDecoder); 670 671 DecoderData& GetDecoderData(TrackType aTrack); 672 673 // Demuxer objects. 674 class DemuxerProxy; 675 UniquePtr<DemuxerProxy> mDemuxer; 676 bool mDemuxerInitDone; 677 void OnDemuxerInitDone(const MediaResult& aResult); 678 void OnDemuxerInitFailed(const MediaResult& aError); 679 MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; 680 MozPromiseRequestHolder<NotifyDataArrivedPromise> mNotifyDataArrivedPromise; 681 bool mPendingNotifyDataArrived; 682 void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); 683 684 void DoDemuxVideo(); 685 void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnVideoDemuxFailed(const MediaResult & aError)686 void OnVideoDemuxFailed(const MediaResult& aError) { 687 OnDemuxFailed(TrackType::kVideoTrack, aError); 688 } 689 690 void DoDemuxAudio(); 691 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnAudioDemuxFailed(const MediaResult & aError)692 void OnAudioDemuxFailed(const MediaResult& aError) { 693 OnDemuxFailed(TrackType::kAudioTrack, aError); 694 } 695 696 void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold); 697 MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> 698 mSkipRequest; 699 void VideoSkipReset(uint32_t aSkipped); 700 void OnVideoSkipCompleted(uint32_t aSkipped); 701 void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure); 702 703 // The last number of decoded output frames that we've reported to 704 // MediaDecoder::NotifyDecoded(). We diff the number of output video 705 // frames every time that DecodeVideoData() is called, and report the 706 // delta there. 707 uint64_t mLastReportedNumDecodedFrames; 708 709 // Timestamp of the previous decoded keyframe, in microseconds. 710 int64_t mPreviousDecodedKeyframeTime_us; 711 // Default mLastDecodedKeyframeTime_us value, must be bigger than anything. 712 static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX; 713 714 RefPtr<layers::KnowsCompositor> mKnowsCompositor; 715 716 // Metadata objects 717 // True if we've read the streams' metadata. 718 bool mInitDone; 719 MozPromiseHolder<MetadataPromise> mMetadataPromise; 720 bool IsEncrypted() const; 721 722 // Set to true if any of our track buffers may be blocking. 723 bool mTrackDemuxersMayBlock; 724 725 // Seeking objects. 726 void SetSeekTarget(const SeekTarget& aTarget); IsSeeking()727 bool IsSeeking() const { return mPendingSeekTime.isSome(); } IsVideoSeeking()728 bool IsVideoSeeking() const { 729 return IsSeeking() && mOriginalSeekTarget.IsVideoOnly(); 730 } 731 void ScheduleSeek(); 732 void AttemptSeek(); 733 void OnSeekFailed(TrackType aTrack, const MediaResult& aError); 734 void DoVideoSeek(); 735 void OnVideoSeekCompleted(media::TimeUnit aTime); 736 void OnVideoSeekFailed(const MediaResult& aError); 737 bool mSeekScheduled; 738 739 void DoAudioSeek(); 740 void OnAudioSeekCompleted(media::TimeUnit aTime); 741 void OnAudioSeekFailed(const MediaResult& aError); 742 // The SeekTarget that was last given to Seek() 743 SeekTarget mOriginalSeekTarget; 744 // Temporary seek information while we wait for the data 745 Maybe<media::TimeUnit> mFallbackSeekTime; 746 Maybe<media::TimeUnit> mPendingSeekTime; 747 MozPromiseHolder<SeekPromise> mSeekPromise; 748 749 RefPtr<VideoFrameContainer> mVideoFrameContainer; 750 layers::ImageContainer* GetImageContainer(); 751 752 RefPtr<CDMProxy> mCDMProxy; 753 754 RefPtr<GMPCrashHelper> mCrashHelper; 755 756 void SetNullDecode(TrackType aTrack, bool aIsNullDecode); 757 758 class DecoderFactory; 759 UniquePtr<DecoderFactory> mDecoderFactory; 760 761 class ShutdownPromisePool; 762 UniquePtr<ShutdownPromisePool> mShutdownPromisePool; 763 764 MediaEventListener mOnTrackWaitingForKeyListener; 765 766 void OnFirstDemuxCompleted(TrackInfo::TrackType aType, 767 RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); 768 769 void OnFirstDemuxFailed(TrackInfo::TrackType aType, 770 const MediaResult& aError); 771 772 void MaybeResolveMetadataPromise(); 773 774 // Stores presentation info required for playback. 775 MediaInfo mInfo; 776 777 UniquePtr<MetadataTags> mTags; 778 779 // A flag indicating if the start time is known or not. 780 bool mHasStartTime = false; 781 782 void ShutdownDecoder(TrackType aTrack); 783 RefPtr<ShutdownPromise> TearDownDecoders(); 784 785 bool mShutdown = false; 786 787 // Buffered range. 788 Canonical<media::TimeIntervals> mBuffered; 789 790 // Used to send TimedMetadata to the listener. 791 TimedMetadataEventProducer mTimedMetadataEvent; 792 793 // Notify if this media is not seekable. 794 MediaEventProducer<void> mOnMediaNotSeekable; 795 796 // Notify if we are waiting for a decryption key. 797 MediaEventProducer<TrackInfo::TrackType> mOnTrackWaitingForKey; 798 799 MediaEventProducer<nsTArray<uint8_t>, nsString> mOnEncrypted; 800 801 MediaEventProducer<void> mOnWaitingForKey; 802 803 MediaEventProducer<MediaResult> mOnDecodeWarning; 804 805 MediaEventProducer<VideoInfo> mOnStoreDecoderBenchmark; 806 807 RefPtr<FrameStatistics> mFrameStats; 808 809 // Used in bug 1393399 for telemetry. 810 const MediaDecoderOwnerID mMediaDecoderOwnerID; 811 812 bool ResolveSetCDMPromiseIfDone(TrackType aTrack); 813 void PrepareToSetCDMForTrack(TrackType aTrack); 814 MozPromiseHolder<SetCDMPromise> mSetCDMPromise; 815 TrackSet mSetCDMForTracks{}; 816 bool IsDecoderWaitingForCDM(TrackType aTrack); 817 }; 818 819 } // namespace mozilla 820 821 #endif 822