1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #if !defined(MediaFormatReader_h_) 8 #define MediaFormatReader_h_ 9 10 #include "mozilla/Atomics.h" 11 #include "mozilla/Maybe.h" 12 #include "mozilla/TaskQueue.h" 13 #include "mozilla/Monitor.h" 14 15 #include "MediaEventSource.h" 16 #include "MediaDataDemuxer.h" 17 #include "MediaDecoderReader.h" 18 #include "nsAutoPtr.h" 19 #include "PDMFactory.h" 20 21 namespace mozilla { 22 23 class CDMProxy; 24 25 class MediaFormatReader final : public MediaDecoderReader 26 { 27 typedef TrackInfo::TrackType TrackType; 28 29 public: 30 MediaFormatReader(AbstractMediaDecoder* aDecoder, 31 MediaDataDemuxer* aDemuxer, 32 VideoFrameContainer* aVideoFrameContainer = nullptr); 33 34 virtual ~MediaFormatReader(); 35 36 size_t SizeOfVideoQueueInFrames() override; 37 size_t SizeOfAudioQueueInFrames() override; 38 39 RefPtr<MediaDataPromise> 40 RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override; 41 42 RefPtr<MediaDataPromise> RequestAudioData() override; 43 44 RefPtr<MetadataPromise> AsyncReadMetadata() override; 45 46 void ReadUpdatedMetadata(MediaInfo* aInfo) override; 47 48 RefPtr<SeekPromise> 49 Seek(SeekTarget aTarget, int64_t aUnused) override; 50 51 protected: 52 void NotifyDataArrivedInternal() override; 53 54 public: 55 media::TimeIntervals GetBuffered() override; 56 57 RefPtr<BufferedUpdatePromise> UpdateBufferedWithPromise() override; 58 59 bool ForceZeroStartTime() const override; 60 61 // For Media Resource Management 62 void ReleaseResources() override; 63 64 nsresult ResetDecode(TrackSet aTracks) override; 65 66 RefPtr<ShutdownPromise> Shutdown() override; 67 IsAsync()68 bool IsAsync() const override { return true; } 69 70 bool VideoIsHardwareAccelerated() const override; 71 IsWaitForDataSupported()72 bool IsWaitForDataSupported() const override { return true; } 73 RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType) override; 74 75 // MediaFormatReader supports demuxed-only mode. IsDemuxOnlySupported()76 bool IsDemuxOnlySupported() const override { return true; } 77 SetDemuxOnly(bool aDemuxedOnly)78 void SetDemuxOnly(bool aDemuxedOnly) override 79 { 80 if (OnTaskQueue()) { 81 mDemuxOnly = aDemuxedOnly; 82 return; 83 } 84 nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>( 85 this, &MediaDecoderReader::SetDemuxOnly, aDemuxedOnly); 86 OwnerThread()->Dispatch(r.forget()); 87 } 88 UseBufferingHeuristics()89 bool UseBufferingHeuristics() const override 90 { 91 return mTrackDemuxersMayBlock; 92 } 93 94 void SetCDMProxy(CDMProxy* aProxy) override; 95 96 // Returns a string describing the state of the decoder data. 97 // Used for debugging purposes. 98 void GetMozDebugReaderData(nsAString& aString); 99 100 void SetVideoBlankDecode(bool aIsBlankDecode) override; 101 102 private: 103 nsresult InitInternal() override; 104 HasVideo()105 bool HasVideo() const { return mVideo.mTrackDemuxer; } HasAudio()106 bool HasAudio() const { return mAudio.mTrackDemuxer; } 107 108 bool IsWaitingOnCDMResource(); 109 110 bool InitDemuxer(); 111 // Notify the demuxer that new data has been received. 112 // The next queued task calling GetBuffered() is guaranteed to have up to date 113 // buffered ranges. 114 void NotifyDemuxer(); 115 void ReturnOutput(MediaData* aData, TrackType aTrack); 116 117 // Enqueues a task to call Update(aTrack) on the decoder task queue. 118 // Lock for corresponding track must be held. 119 void ScheduleUpdate(TrackType aTrack); 120 void Update(TrackType aTrack); 121 // Handle actions should more data be received. 122 // Returns true if no more action is required. 123 bool UpdateReceivedNewData(TrackType aTrack); 124 // Called when new samples need to be demuxed. 125 void RequestDemuxSamples(TrackType aTrack); 126 // Handle demuxed samples by the input behavior. 127 void HandleDemuxedSamples(TrackType aTrack, 128 AbstractMediaDecoder::AutoNotifyDecoded& aA); 129 // Decode any pending already demuxed samples. 130 void DecodeDemuxedSamples(TrackType aTrack, 131 MediaRawData* aSample); 132 133 struct InternalSeekTarget { InternalSeekTargetInternalSeekTarget134 InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget) 135 : mTime(aTime) 136 , mDropTarget(aDropTarget) 137 , mWaiting(false) 138 , mHasSeeked(false) 139 {} 140 TimeInternalSeekTarget141 media::TimeUnit Time() const { return mTime.mStart; } EndTimeInternalSeekTarget142 media::TimeUnit EndTime() const { return mTime.mEnd; } ContainsInternalSeekTarget143 bool Contains(const media::TimeUnit& aTime) const 144 { 145 return mTime.Contains(aTime); 146 } 147 148 media::TimeInterval mTime; 149 bool mDropTarget; 150 bool mWaiting; 151 bool mHasSeeked; 152 }; 153 154 // Perform an internal seek to aTime. If aDropTarget is true then 155 // the first sample past the target will be dropped. 156 void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget); 157 158 // Drain the current decoder. 159 void DrainDecoder(TrackType aTrack); 160 void NotifyNewOutput(TrackType aTrack, MediaData* aSample); 161 void NotifyInputExhausted(TrackType aTrack); 162 void NotifyDrainComplete(TrackType aTrack); 163 void NotifyError(TrackType aTrack, const MediaResult& aError); 164 void NotifyWaitingForData(TrackType aTrack); 165 void NotifyWaitingForKey(TrackType aTrack); 166 void NotifyEndOfStream(TrackType aTrack); 167 168 void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData); 169 170 // Initializes mLayersBackendType if possible. 171 void InitLayersBackendType(); 172 173 // DecoderCallback proxies the MediaDataDecoderCallback calls to these 174 // functions. 175 void Output(TrackType aType, MediaData* aSample); 176 void InputExhausted(TrackType aTrack); 177 void Error(TrackType aTrack, const MediaResult& aError); 178 void Reset(TrackType aTrack); 179 void DrainComplete(TrackType aTrack); 180 void DropDecodedSamples(TrackType aTrack); 181 void WaitingForKey(TrackType aTrack); 182 183 bool ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThreshold); 184 185 void SetVideoDecodeThreshold(); 186 187 size_t SizeOfQueue(TrackType aTrack); 188 189 RefPtr<PDMFactory> mPlatform; 190 191 class DecoderCallback : public MediaDataDecoderCallback { 192 public: DecoderCallback(MediaFormatReader * aReader,TrackType aType)193 DecoderCallback(MediaFormatReader* aReader, TrackType aType) 194 : mReader(aReader) 195 , mType(aType) 196 { 197 } Output(MediaData * aSample)198 void Output(MediaData* aSample) override { 199 mReader->Output(mType, aSample); 200 } InputExhausted()201 void InputExhausted() override { 202 mReader->InputExhausted(mType); 203 } Error(const MediaResult & aError)204 void Error(const MediaResult& aError) override { 205 mReader->Error(mType, aError); 206 } DrainComplete()207 void DrainComplete() override { 208 mReader->DrainComplete(mType); 209 } ReleaseMediaResources()210 void ReleaseMediaResources() override { 211 mReader->ReleaseResources(); 212 } OnReaderTaskQueue()213 bool OnReaderTaskQueue() override { 214 return mReader->OnTaskQueue(); 215 } WaitingForKey()216 void WaitingForKey() override { 217 mReader->WaitingForKey(mType); 218 } 219 220 private: 221 MediaFormatReader* mReader; 222 TrackType mType; 223 }; 224 225 struct DecoderData { DecoderDataDecoderData226 DecoderData(MediaFormatReader* aOwner, 227 MediaData::Type aType, 228 uint32_t aNumOfMaxError) 229 : mOwner(aOwner) 230 , mType(aType) 231 , mMonitor("DecoderData") 232 , mDescription("shutdown") 233 , mUpdateScheduled(false) 234 , mDemuxEOS(false) 235 , mWaitingForData(false) 236 , mWaitingForKey(false) 237 , mReceivedNewData(false) 238 , mOutputRequested(false) 239 , mDecodePending(false) 240 , mNeedDraining(false) 241 , mDraining(false) 242 , mDrainComplete(false) 243 , mNumOfConsecutiveError(0) 244 , mMaxConsecutiveError(aNumOfMaxError) 245 , mNumSamplesInput(0) 246 , mNumSamplesOutput(0) 247 , mNumSamplesOutputTotal(0) 248 , mNumSamplesSkippedTotal(0) 249 , mSizeOfQueue(0) 250 , mIsHardwareAccelerated(false) 251 , mLastStreamSourceID(UINT32_MAX) 252 , mIsBlankDecode(false) 253 {} 254 255 MediaFormatReader* mOwner; 256 // Disambiguate Audio vs Video. 257 MediaData::Type mType; 258 RefPtr<MediaTrackDemuxer> mTrackDemuxer; 259 // TaskQueue on which decoder can choose to decode. 260 // Only non-null up until the decoder is created. 261 RefPtr<TaskQueue> mTaskQueue; 262 // Callback that receives output and error notifications from the decoder. 263 nsAutoPtr<DecoderCallback> mCallback; 264 265 // Monitor protecting mDescription and mDecoder. 266 Monitor mMonitor; 267 // The platform decoder. 268 RefPtr<MediaDataDecoder> mDecoder; 269 const char* mDescription; ShutdownDecoderDecoderData270 void ShutdownDecoder() 271 { 272 MonitorAutoLock mon(mMonitor); 273 if (mDecoder) { 274 mDecoder->Shutdown(); 275 } 276 mDescription = "shutdown"; 277 mDecoder = nullptr; 278 } 279 280 // Only accessed from reader's task queue. 281 bool mUpdateScheduled; 282 bool mDemuxEOS; 283 bool mWaitingForData; 284 bool mWaitingForKey; 285 bool mReceivedNewData; 286 287 // Pending seek. 288 MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest; 289 290 // Queued demux samples waiting to be decoded. 291 nsTArray<RefPtr<MediaRawData>> mQueuedSamples; 292 MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; 293 // A WaitingPromise is pending if the demuxer is waiting for data or 294 // if the decoder is waiting for a key. 295 MozPromiseHolder<WaitForDataPromise> mWaitingPromise; HasWaitingPromiseDecoderData296 bool HasWaitingPromise() const 297 { 298 MOZ_ASSERT(mOwner->OnTaskQueue()); 299 return !mWaitingPromise.IsEmpty(); 300 } IsWaitingDecoderData301 bool IsWaiting() const 302 { 303 MOZ_ASSERT(mOwner->OnTaskQueue()); 304 return mWaitingForData || mWaitingForKey; 305 } 306 307 // MediaDataDecoder handler's variables. 308 bool mOutputRequested; 309 // Set to true once the MediaDataDecoder has been fed a compressed sample. 310 // No more samples will be passed to the decoder while true. 311 // mDecodePending is reset when: 312 // 1- The decoder calls InputExhausted 313 // 2- The decoder is Flushed or Reset. 314 bool mDecodePending; 315 bool mNeedDraining; 316 bool mDraining; 317 bool mDrainComplete; 318 HasPendingDrainDecoderData319 bool HasPendingDrain() const 320 { 321 return mDraining || mDrainComplete; 322 } 323 324 uint32_t mNumOfConsecutiveError; 325 uint32_t mMaxConsecutiveError; 326 327 Maybe<MediaResult> mError; HasFatalErrorDecoderData328 bool HasFatalError() const 329 { 330 if (!mError.isSome()) { 331 return false; 332 } 333 if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) { 334 // Allow decode errors to be non-fatal, but give up 335 // if we have too many. 336 return mNumOfConsecutiveError > mMaxConsecutiveError; 337 } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) { 338 // If the caller asked for a new decoder we shouldn't treat 339 // it as fatal. 340 return false; 341 } else { 342 // All other error types are fatal 343 return true; 344 } 345 } 346 347 // If set, all decoded samples prior mTimeThreshold will be dropped. 348 // Used for internal seeking when a change of stream is detected or when 349 // encountering data discontinuity. 350 Maybe<InternalSeekTarget> mTimeThreshold; 351 // Time of last sample returned. 352 Maybe<media::TimeInterval> mLastSampleTime; 353 354 // Decoded samples returned my mDecoder awaiting being returned to 355 // state machine upon request. 356 nsTArray<RefPtr<MediaData>> mOutput; 357 uint64_t mNumSamplesInput; 358 uint64_t mNumSamplesOutput; 359 uint64_t mNumSamplesOutputTotal; 360 uint64_t mNumSamplesSkippedTotal; 361 362 // These get overridden in the templated concrete class. 363 // Indicate if we have a pending promise for decoded frame. 364 // Rejecting the promise will stop the reader from decoding ahead. 365 virtual bool HasPromise() const = 0; 366 virtual RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) = 0; 367 virtual void ResolvePromise(MediaData* aData, const char* aMethodName) = 0; 368 virtual void RejectPromise(const MediaResult& aError, 369 const char* aMethodName) = 0; 370 371 // Clear track demuxer related data. ResetDemuxerDecoderData372 void ResetDemuxer() 373 { 374 mDemuxRequest.DisconnectIfExists(); 375 mSeekRequest.DisconnectIfExists(); 376 mTrackDemuxer->Reset(); 377 mQueuedSamples.Clear(); 378 } 379 380 // Flush the decoder if present and reset decoding related data. 381 // Decoding will be suspended until mInputRequested is set again. 382 // Following a flush, the decoder is ready to accept any new data. FlushDecoderData383 void Flush() 384 { 385 if (mDecoder) { 386 mDecoder->Flush(); 387 } 388 mOutputRequested = false; 389 mDecodePending = false; 390 mOutput.Clear(); 391 mNumSamplesInput = 0; 392 mNumSamplesOutput = 0; 393 mSizeOfQueue = 0; 394 mDraining = false; 395 mDrainComplete = false; 396 } 397 398 // Reset the state of the DecoderData, clearing all queued frames 399 // (pending demuxed and decoded). 400 // Decoding will be suspended until mInputRequested is set again. 401 // The track demuxer is *not* reset. ResetStateDecoderData402 void ResetState() 403 { 404 MOZ_ASSERT(mOwner->OnTaskQueue()); 405 mDemuxEOS = false; 406 mWaitingForData = false; 407 mWaitingForKey = false; 408 mQueuedSamples.Clear(); 409 mOutputRequested = false; 410 mNeedDraining = false; 411 mDecodePending = false; 412 mDraining = false; 413 mDrainComplete = false; 414 mTimeThreshold.reset(); 415 mLastSampleTime.reset(); 416 mOutput.Clear(); 417 mNumSamplesInput = 0; 418 mNumSamplesOutput = 0; 419 mSizeOfQueue = 0; 420 mNextStreamSourceID.reset(); 421 if (!HasFatalError()) { 422 mError.reset(); 423 } 424 } 425 HasInternalSeekPendingDecoderData426 bool HasInternalSeekPending() const 427 { 428 return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked; 429 } 430 431 // Used by the MDSM for logging purposes. 432 Atomic<size_t> mSizeOfQueue; 433 // Used by the MDSM to determine if video decoding is hardware accelerated. 434 // This value is updated after a frame is successfully decoded. 435 Atomic<bool> mIsHardwareAccelerated; 436 // Sample format monitoring. 437 uint32_t mLastStreamSourceID; 438 Maybe<uint32_t> mNextStreamSourceID; 439 media::TimeIntervals mTimeRanges; 440 Maybe<media::TimeUnit> mLastTimeRangesEnd; 441 // TrackInfo as first discovered during ReadMetadata. 442 UniquePtr<TrackInfo> mOriginalInfo; 443 RefPtr<SharedTrackInfo> mInfo; 444 Maybe<media::TimeUnit> mFirstDemuxedSampleTime; 445 // Use BlankDecoderModule or not. 446 bool mIsBlankDecode; 447 448 }; 449 450 class DecoderDataWithPromise : public DecoderData { 451 public: DecoderDataWithPromise(MediaFormatReader * aOwner,MediaData::Type aType,uint32_t aNumOfMaxError)452 DecoderDataWithPromise(MediaFormatReader* aOwner, 453 MediaData::Type aType, 454 uint32_t aNumOfMaxError) 455 : DecoderData(aOwner, aType, aNumOfMaxError) 456 , mHasPromise(false) 457 458 {} 459 HasPromise()460 bool HasPromise() const override 461 { 462 return mHasPromise; 463 } 464 EnsurePromise(const char * aMethodName)465 RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) override 466 { 467 MOZ_ASSERT(mOwner->OnTaskQueue()); 468 mHasPromise = true; 469 return mPromise.Ensure(aMethodName); 470 } 471 ResolvePromise(MediaData * aData,const char * aMethodName)472 void ResolvePromise(MediaData* aData, const char* aMethodName) override 473 { 474 MOZ_ASSERT(mOwner->OnTaskQueue()); 475 mPromise.Resolve(aData, aMethodName); 476 mHasPromise = false; 477 } 478 RejectPromise(const MediaResult & aError,const char * aMethodName)479 void RejectPromise(const MediaResult& aError, 480 const char* aMethodName) override 481 { 482 MOZ_ASSERT(mOwner->OnTaskQueue()); 483 mPromise.Reject(aError, aMethodName); 484 mHasPromise = false; 485 } 486 487 private: 488 MozPromiseHolder<MediaDataPromise> mPromise; 489 Atomic<bool> mHasPromise; 490 }; 491 492 DecoderDataWithPromise mAudio; 493 DecoderDataWithPromise mVideo; 494 495 // Returns true when the decoder for this track needs input. 496 bool NeedInput(DecoderData& aDecoder); 497 498 DecoderData& GetDecoderData(TrackType aTrack); 499 500 // Demuxer objects. 501 RefPtr<MediaDataDemuxer> mDemuxer; 502 bool mDemuxerInitDone; 503 void OnDemuxerInitDone(nsresult); 504 void OnDemuxerInitFailed(const MediaResult& aError); 505 MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; 506 void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); 507 508 void DoDemuxVideo(); 509 void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnVideoDemuxFailed(const MediaResult & aError)510 void OnVideoDemuxFailed(const MediaResult& aError) 511 { 512 OnDemuxFailed(TrackType::kVideoTrack, aError); 513 } 514 515 void DoDemuxAudio(); 516 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnAudioDemuxFailed(const MediaResult & aError)517 void OnAudioDemuxFailed(const MediaResult& aError) 518 { 519 OnDemuxFailed(TrackType::kAudioTrack, aError); 520 } 521 522 void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold); 523 MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> mSkipRequest; 524 void VideoSkipReset(uint32_t aSkipped); 525 void OnVideoSkipCompleted(uint32_t aSkipped); 526 void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure); 527 528 // The last number of decoded output frames that we've reported to 529 // MediaDecoder::NotifyDecoded(). We diff the number of output video 530 // frames every time that DecodeVideoData() is called, and report the 531 // delta there. 532 uint64_t mLastReportedNumDecodedFrames; 533 534 // Timestamp of the previous decoded keyframe, in microseconds. 535 int64_t mPreviousDecodedKeyframeTime_us; 536 // Default mLastDecodedKeyframeTime_us value, must be bigger than anything. 537 static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX; 538 539 RefPtr<layers::KnowsCompositor> mKnowsCompositor; 540 541 // Metadata objects 542 // True if we've read the streams' metadata. 543 bool mInitDone; 544 MozPromiseHolder<MetadataPromise> mMetadataPromise; 545 bool IsEncrypted() const; 546 547 // Set to true if any of our track buffers may be blocking. 548 bool mTrackDemuxersMayBlock; 549 550 // Set the demuxed-only flag. 551 Atomic<bool> mDemuxOnly; 552 553 // Seeking objects. 554 void SetSeekTarget(const SeekTarget& aTarget); 555 media::TimeUnit DemuxStartTime(); IsSeeking()556 bool IsSeeking() const { return mPendingSeekTime.isSome(); } IsVideoSeeking()557 bool IsVideoSeeking() const 558 { 559 return IsSeeking() && mOriginalSeekTarget.IsVideoOnly(); 560 } 561 void ScheduleSeek(); 562 void AttemptSeek(); 563 void OnSeekFailed(TrackType aTrack, const MediaResult& aError); 564 void DoVideoSeek(); 565 void OnVideoSeekCompleted(media::TimeUnit aTime); 566 void OnVideoSeekFailed(const MediaResult& aError); 567 bool mSeekScheduled; 568 NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)569 void NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor) 570 { 571 mKnowsCompositor = aKnowsCompositor; 572 } 573 574 void DoAudioSeek(); 575 void OnAudioSeekCompleted(media::TimeUnit aTime); 576 void OnAudioSeekFailed(const MediaResult& aError); 577 // The SeekTarget that was last given to Seek() 578 SeekTarget mOriginalSeekTarget; 579 // Temporary seek information while we wait for the data 580 Maybe<media::TimeUnit> mFallbackSeekTime; 581 Maybe<media::TimeUnit> mPendingSeekTime; 582 MozPromiseHolder<SeekPromise> mSeekPromise; 583 584 RefPtr<VideoFrameContainer> mVideoFrameContainer; 585 layers::ImageContainer* GetImageContainer(); 586 587 RefPtr<CDMProxy> mCDMProxy; 588 589 RefPtr<GMPCrashHelper> mCrashHelper; 590 591 void SetBlankDecode(TrackType aTrack, bool aIsBlankDecode); 592 593 class DecoderFactory; 594 UniquePtr<DecoderFactory> mDecoderFactory; 595 596 MediaEventListener mCompositorUpdatedListener; 597 }; 598 599 } // namespace mozilla 600 601 #endif 602