1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 
7 #if !defined(MediaFormatReader_h_)
8 #define MediaFormatReader_h_
9 
10 #include "mozilla/Atomics.h"
11 #include "mozilla/Maybe.h"
12 #include "mozilla/TaskQueue.h"
13 #include "mozilla/Monitor.h"
14 
15 #include "MediaEventSource.h"
16 #include "MediaDataDemuxer.h"
17 #include "MediaDecoderReader.h"
18 #include "nsAutoPtr.h"
19 #include "PDMFactory.h"
20 
21 namespace mozilla {
22 
23 class CDMProxy;
24 
25 class MediaFormatReader final : public MediaDecoderReader
26 {
27   typedef TrackInfo::TrackType TrackType;
28 
29 public:
30   MediaFormatReader(AbstractMediaDecoder* aDecoder,
31                     MediaDataDemuxer* aDemuxer,
32                     VideoFrameContainer* aVideoFrameContainer = nullptr);
33 
34   virtual ~MediaFormatReader();
35 
36   size_t SizeOfVideoQueueInFrames() override;
37   size_t SizeOfAudioQueueInFrames() override;
38 
39   RefPtr<MediaDataPromise>
40   RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override;
41 
42   RefPtr<MediaDataPromise> RequestAudioData() override;
43 
44   RefPtr<MetadataPromise> AsyncReadMetadata() override;
45 
46   void ReadUpdatedMetadata(MediaInfo* aInfo) override;
47 
48   RefPtr<SeekPromise>
49   Seek(SeekTarget aTarget, int64_t aUnused) override;
50 
51 protected:
52   void NotifyDataArrivedInternal() override;
53 
54 public:
55   media::TimeIntervals GetBuffered() override;
56 
57   RefPtr<BufferedUpdatePromise> UpdateBufferedWithPromise() override;
58 
59   bool ForceZeroStartTime() const override;
60 
61   // For Media Resource Management
62   void ReleaseResources() override;
63 
64   nsresult ResetDecode(TrackSet aTracks) override;
65 
66   RefPtr<ShutdownPromise> Shutdown() override;
67 
IsAsync()68   bool IsAsync() const override { return true; }
69 
70   bool VideoIsHardwareAccelerated() const override;
71 
IsWaitForDataSupported()72   bool IsWaitForDataSupported() const override { return true; }
73   RefPtr<WaitForDataPromise> WaitForData(MediaData::Type aType) override;
74 
75   // MediaFormatReader supports demuxed-only mode.
IsDemuxOnlySupported()76   bool IsDemuxOnlySupported() const override { return true; }
77 
SetDemuxOnly(bool aDemuxedOnly)78   void SetDemuxOnly(bool aDemuxedOnly) override
79   {
80     if (OnTaskQueue()) {
81       mDemuxOnly = aDemuxedOnly;
82       return;
83     }
84     nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>(
85       this, &MediaDecoderReader::SetDemuxOnly, aDemuxedOnly);
86     OwnerThread()->Dispatch(r.forget());
87   }
88 
UseBufferingHeuristics()89   bool UseBufferingHeuristics() const override
90   {
91     return mTrackDemuxersMayBlock;
92   }
93 
94   void SetCDMProxy(CDMProxy* aProxy) override;
95 
96   // Returns a string describing the state of the decoder data.
97   // Used for debugging purposes.
98   void GetMozDebugReaderData(nsAString& aString);
99 
100   void SetVideoBlankDecode(bool aIsBlankDecode) override;
101 
102 private:
103   nsresult InitInternal() override;
104 
HasVideo()105   bool HasVideo() const { return mVideo.mTrackDemuxer; }
HasAudio()106   bool HasAudio() const { return mAudio.mTrackDemuxer; }
107 
108   bool IsWaitingOnCDMResource();
109 
110   bool InitDemuxer();
111   // Notify the demuxer that new data has been received.
112   // The next queued task calling GetBuffered() is guaranteed to have up to date
113   // buffered ranges.
114   void NotifyDemuxer();
115   void ReturnOutput(MediaData* aData, TrackType aTrack);
116 
117   // Enqueues a task to call Update(aTrack) on the decoder task queue.
118   // Lock for corresponding track must be held.
119   void ScheduleUpdate(TrackType aTrack);
120   void Update(TrackType aTrack);
121   // Handle actions should more data be received.
122   // Returns true if no more action is required.
123   bool UpdateReceivedNewData(TrackType aTrack);
124   // Called when new samples need to be demuxed.
125   void RequestDemuxSamples(TrackType aTrack);
126   // Handle demuxed samples by the input behavior.
127   void HandleDemuxedSamples(TrackType aTrack,
128                             AbstractMediaDecoder::AutoNotifyDecoded& aA);
129   // Decode any pending already demuxed samples.
130   void DecodeDemuxedSamples(TrackType aTrack,
131                             MediaRawData* aSample);
132 
133   struct InternalSeekTarget {
InternalSeekTargetInternalSeekTarget134     InternalSeekTarget(const media::TimeInterval& aTime, bool aDropTarget)
135       : mTime(aTime)
136       , mDropTarget(aDropTarget)
137       , mWaiting(false)
138       , mHasSeeked(false)
139     {}
140 
TimeInternalSeekTarget141     media::TimeUnit Time() const { return mTime.mStart; }
EndTimeInternalSeekTarget142     media::TimeUnit EndTime() const { return mTime.mEnd; }
ContainsInternalSeekTarget143     bool Contains(const media::TimeUnit& aTime) const
144     {
145       return mTime.Contains(aTime);
146     }
147 
148     media::TimeInterval mTime;
149     bool mDropTarget;
150     bool mWaiting;
151     bool mHasSeeked;
152   };
153 
154   // Perform an internal seek to aTime. If aDropTarget is true then
155   // the first sample past the target will be dropped.
156   void InternalSeek(TrackType aTrack, const InternalSeekTarget& aTarget);
157 
158   // Drain the current decoder.
159   void DrainDecoder(TrackType aTrack);
160   void NotifyNewOutput(TrackType aTrack, MediaData* aSample);
161   void NotifyInputExhausted(TrackType aTrack);
162   void NotifyDrainComplete(TrackType aTrack);
163   void NotifyError(TrackType aTrack, const MediaResult& aError);
164   void NotifyWaitingForData(TrackType aTrack);
165   void NotifyWaitingForKey(TrackType aTrack);
166   void NotifyEndOfStream(TrackType aTrack);
167 
168   void ExtractCryptoInitData(nsTArray<uint8_t>& aInitData);
169 
170   // Initializes mLayersBackendType if possible.
171   void InitLayersBackendType();
172 
173   // DecoderCallback proxies the MediaDataDecoderCallback calls to these
174   // functions.
175   void Output(TrackType aType, MediaData* aSample);
176   void InputExhausted(TrackType aTrack);
177   void Error(TrackType aTrack, const MediaResult& aError);
178   void Reset(TrackType aTrack);
179   void DrainComplete(TrackType aTrack);
180   void DropDecodedSamples(TrackType aTrack);
181   void WaitingForKey(TrackType aTrack);
182 
183   bool ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThreshold);
184 
185   void SetVideoDecodeThreshold();
186 
187   size_t SizeOfQueue(TrackType aTrack);
188 
189   RefPtr<PDMFactory> mPlatform;
190 
191   class DecoderCallback : public MediaDataDecoderCallback {
192   public:
DecoderCallback(MediaFormatReader * aReader,TrackType aType)193     DecoderCallback(MediaFormatReader* aReader, TrackType aType)
194       : mReader(aReader)
195       , mType(aType)
196     {
197     }
Output(MediaData * aSample)198     void Output(MediaData* aSample) override {
199       mReader->Output(mType, aSample);
200     }
InputExhausted()201     void InputExhausted() override {
202       mReader->InputExhausted(mType);
203     }
Error(const MediaResult & aError)204     void Error(const MediaResult& aError) override {
205       mReader->Error(mType, aError);
206     }
DrainComplete()207     void DrainComplete() override {
208       mReader->DrainComplete(mType);
209     }
ReleaseMediaResources()210     void ReleaseMediaResources() override {
211       mReader->ReleaseResources();
212     }
OnReaderTaskQueue()213     bool OnReaderTaskQueue() override {
214       return mReader->OnTaskQueue();
215     }
WaitingForKey()216     void WaitingForKey() override {
217       mReader->WaitingForKey(mType);
218     }
219 
220   private:
221     MediaFormatReader* mReader;
222     TrackType mType;
223   };
224 
225   struct DecoderData {
DecoderDataDecoderData226     DecoderData(MediaFormatReader* aOwner,
227                 MediaData::Type aType,
228                 uint32_t aNumOfMaxError)
229       : mOwner(aOwner)
230       , mType(aType)
231       , mMonitor("DecoderData")
232       , mDescription("shutdown")
233       , mUpdateScheduled(false)
234       , mDemuxEOS(false)
235       , mWaitingForData(false)
236       , mWaitingForKey(false)
237       , mReceivedNewData(false)
238       , mOutputRequested(false)
239       , mDecodePending(false)
240       , mNeedDraining(false)
241       , mDraining(false)
242       , mDrainComplete(false)
243       , mNumOfConsecutiveError(0)
244       , mMaxConsecutiveError(aNumOfMaxError)
245       , mNumSamplesInput(0)
246       , mNumSamplesOutput(0)
247       , mNumSamplesOutputTotal(0)
248       , mNumSamplesSkippedTotal(0)
249       , mSizeOfQueue(0)
250       , mIsHardwareAccelerated(false)
251       , mLastStreamSourceID(UINT32_MAX)
252       , mIsBlankDecode(false)
253     {}
254 
255     MediaFormatReader* mOwner;
256     // Disambiguate Audio vs Video.
257     MediaData::Type mType;
258     RefPtr<MediaTrackDemuxer> mTrackDemuxer;
259     // TaskQueue on which decoder can choose to decode.
260     // Only non-null up until the decoder is created.
261     RefPtr<TaskQueue> mTaskQueue;
262     // Callback that receives output and error notifications from the decoder.
263     nsAutoPtr<DecoderCallback> mCallback;
264 
265     // Monitor protecting mDescription and mDecoder.
266     Monitor mMonitor;
267     // The platform decoder.
268     RefPtr<MediaDataDecoder> mDecoder;
269     const char* mDescription;
ShutdownDecoderDecoderData270     void ShutdownDecoder()
271     {
272       MonitorAutoLock mon(mMonitor);
273       if (mDecoder) {
274         mDecoder->Shutdown();
275       }
276       mDescription = "shutdown";
277       mDecoder = nullptr;
278     }
279 
280     // Only accessed from reader's task queue.
281     bool mUpdateScheduled;
282     bool mDemuxEOS;
283     bool mWaitingForData;
284     bool mWaitingForKey;
285     bool mReceivedNewData;
286 
287     // Pending seek.
288     MozPromiseRequestHolder<MediaTrackDemuxer::SeekPromise> mSeekRequest;
289 
290     // Queued demux samples waiting to be decoded.
291     nsTArray<RefPtr<MediaRawData>> mQueuedSamples;
292     MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
293     // A WaitingPromise is pending if the demuxer is waiting for data or
294     // if the decoder is waiting for a key.
295     MozPromiseHolder<WaitForDataPromise> mWaitingPromise;
HasWaitingPromiseDecoderData296     bool HasWaitingPromise() const
297     {
298       MOZ_ASSERT(mOwner->OnTaskQueue());
299       return !mWaitingPromise.IsEmpty();
300     }
IsWaitingDecoderData301     bool IsWaiting() const
302     {
303       MOZ_ASSERT(mOwner->OnTaskQueue());
304       return mWaitingForData || mWaitingForKey;
305     }
306 
307     // MediaDataDecoder handler's variables.
308     bool mOutputRequested;
309     // Set to true once the MediaDataDecoder has been fed a compressed sample.
310     // No more samples will be passed to the decoder while true.
311     // mDecodePending is reset when:
312     // 1- The decoder calls InputExhausted
313     // 2- The decoder is Flushed or Reset.
314     bool mDecodePending;
315     bool mNeedDraining;
316     bool mDraining;
317     bool mDrainComplete;
318 
HasPendingDrainDecoderData319     bool HasPendingDrain() const
320     {
321       return mDraining || mDrainComplete;
322     }
323 
324     uint32_t mNumOfConsecutiveError;
325     uint32_t mMaxConsecutiveError;
326 
327     Maybe<MediaResult> mError;
HasFatalErrorDecoderData328     bool HasFatalError() const
329     {
330       if (!mError.isSome()) {
331         return false;
332       }
333       if (mError.ref() == NS_ERROR_DOM_MEDIA_DECODE_ERR) {
334         // Allow decode errors to be non-fatal, but give up
335         // if we have too many.
336         return mNumOfConsecutiveError > mMaxConsecutiveError;
337       } else if (mError.ref() == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
338         // If the caller asked for a new decoder we shouldn't treat
339         // it as fatal.
340         return false;
341       } else {
342         // All other error types are fatal
343         return true;
344       }
345     }
346 
347     // If set, all decoded samples prior mTimeThreshold will be dropped.
348     // Used for internal seeking when a change of stream is detected or when
349     // encountering data discontinuity.
350     Maybe<InternalSeekTarget> mTimeThreshold;
351     // Time of last sample returned.
352     Maybe<media::TimeInterval> mLastSampleTime;
353 
354     // Decoded samples returned my mDecoder awaiting being returned to
355     // state machine upon request.
356     nsTArray<RefPtr<MediaData>> mOutput;
357     uint64_t mNumSamplesInput;
358     uint64_t mNumSamplesOutput;
359     uint64_t mNumSamplesOutputTotal;
360     uint64_t mNumSamplesSkippedTotal;
361 
362     // These get overridden in the templated concrete class.
363     // Indicate if we have a pending promise for decoded frame.
364     // Rejecting the promise will stop the reader from decoding ahead.
365     virtual bool HasPromise() const = 0;
366     virtual RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) = 0;
367     virtual void ResolvePromise(MediaData* aData, const char* aMethodName) = 0;
368     virtual void RejectPromise(const MediaResult& aError,
369                                const char* aMethodName) = 0;
370 
371     // Clear track demuxer related data.
ResetDemuxerDecoderData372     void ResetDemuxer()
373     {
374       mDemuxRequest.DisconnectIfExists();
375       mSeekRequest.DisconnectIfExists();
376       mTrackDemuxer->Reset();
377       mQueuedSamples.Clear();
378     }
379 
380     // Flush the decoder if present and reset decoding related data.
381     // Decoding will be suspended until mInputRequested is set again.
382     // Following a flush, the decoder is ready to accept any new data.
FlushDecoderData383     void Flush()
384     {
385       if (mDecoder) {
386         mDecoder->Flush();
387       }
388       mOutputRequested = false;
389       mDecodePending = false;
390       mOutput.Clear();
391       mNumSamplesInput = 0;
392       mNumSamplesOutput = 0;
393       mSizeOfQueue = 0;
394       mDraining = false;
395       mDrainComplete = false;
396     }
397 
398     // Reset the state of the DecoderData, clearing all queued frames
399     // (pending demuxed and decoded).
400     // Decoding will be suspended until mInputRequested is set again.
401     // The track demuxer is *not* reset.
ResetStateDecoderData402     void ResetState()
403     {
404       MOZ_ASSERT(mOwner->OnTaskQueue());
405       mDemuxEOS = false;
406       mWaitingForData = false;
407       mWaitingForKey = false;
408       mQueuedSamples.Clear();
409       mOutputRequested = false;
410       mNeedDraining = false;
411       mDecodePending = false;
412       mDraining = false;
413       mDrainComplete = false;
414       mTimeThreshold.reset();
415       mLastSampleTime.reset();
416       mOutput.Clear();
417       mNumSamplesInput = 0;
418       mNumSamplesOutput = 0;
419       mSizeOfQueue = 0;
420       mNextStreamSourceID.reset();
421       if (!HasFatalError()) {
422         mError.reset();
423       }
424     }
425 
HasInternalSeekPendingDecoderData426     bool HasInternalSeekPending() const
427     {
428       return mTimeThreshold && !mTimeThreshold.ref().mHasSeeked;
429     }
430 
431     // Used by the MDSM for logging purposes.
432     Atomic<size_t> mSizeOfQueue;
433     // Used by the MDSM to determine if video decoding is hardware accelerated.
434     // This value is updated after a frame is successfully decoded.
435     Atomic<bool> mIsHardwareAccelerated;
436     // Sample format monitoring.
437     uint32_t mLastStreamSourceID;
438     Maybe<uint32_t> mNextStreamSourceID;
439     media::TimeIntervals mTimeRanges;
440     Maybe<media::TimeUnit> mLastTimeRangesEnd;
441     // TrackInfo as first discovered during ReadMetadata.
442     UniquePtr<TrackInfo> mOriginalInfo;
443     RefPtr<SharedTrackInfo> mInfo;
444     Maybe<media::TimeUnit> mFirstDemuxedSampleTime;
445     // Use BlankDecoderModule or not.
446     bool mIsBlankDecode;
447 
448   };
449 
450   class DecoderDataWithPromise : public DecoderData {
451   public:
DecoderDataWithPromise(MediaFormatReader * aOwner,MediaData::Type aType,uint32_t aNumOfMaxError)452     DecoderDataWithPromise(MediaFormatReader* aOwner,
453                            MediaData::Type aType,
454                            uint32_t aNumOfMaxError)
455       : DecoderData(aOwner, aType, aNumOfMaxError)
456       , mHasPromise(false)
457 
458     {}
459 
HasPromise()460     bool HasPromise() const override
461     {
462       return mHasPromise;
463     }
464 
EnsurePromise(const char * aMethodName)465     RefPtr<MediaDataPromise> EnsurePromise(const char* aMethodName) override
466     {
467       MOZ_ASSERT(mOwner->OnTaskQueue());
468       mHasPromise = true;
469       return mPromise.Ensure(aMethodName);
470     }
471 
ResolvePromise(MediaData * aData,const char * aMethodName)472     void ResolvePromise(MediaData* aData, const char* aMethodName) override
473     {
474       MOZ_ASSERT(mOwner->OnTaskQueue());
475       mPromise.Resolve(aData, aMethodName);
476       mHasPromise = false;
477     }
478 
RejectPromise(const MediaResult & aError,const char * aMethodName)479     void RejectPromise(const MediaResult& aError,
480                        const char* aMethodName) override
481     {
482       MOZ_ASSERT(mOwner->OnTaskQueue());
483       mPromise.Reject(aError, aMethodName);
484       mHasPromise = false;
485     }
486 
487   private:
488     MozPromiseHolder<MediaDataPromise> mPromise;
489     Atomic<bool> mHasPromise;
490   };
491 
492   DecoderDataWithPromise mAudio;
493   DecoderDataWithPromise mVideo;
494 
495   // Returns true when the decoder for this track needs input.
496   bool NeedInput(DecoderData& aDecoder);
497 
498   DecoderData& GetDecoderData(TrackType aTrack);
499 
500   // Demuxer objects.
501   RefPtr<MediaDataDemuxer> mDemuxer;
502   bool mDemuxerInitDone;
503   void OnDemuxerInitDone(nsresult);
504   void OnDemuxerInitFailed(const MediaResult& aError);
505   MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
506   void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
507 
508   void DoDemuxVideo();
509   void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
OnVideoDemuxFailed(const MediaResult & aError)510   void OnVideoDemuxFailed(const MediaResult& aError)
511   {
512     OnDemuxFailed(TrackType::kVideoTrack, aError);
513   }
514 
515   void DoDemuxAudio();
516   void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
OnAudioDemuxFailed(const MediaResult & aError)517   void OnAudioDemuxFailed(const MediaResult& aError)
518   {
519     OnDemuxFailed(TrackType::kAudioTrack, aError);
520   }
521 
522   void SkipVideoDemuxToNextKeyFrame(media::TimeUnit aTimeThreshold);
523   MozPromiseRequestHolder<MediaTrackDemuxer::SkipAccessPointPromise> mSkipRequest;
524   void VideoSkipReset(uint32_t aSkipped);
525   void OnVideoSkipCompleted(uint32_t aSkipped);
526   void OnVideoSkipFailed(MediaTrackDemuxer::SkipFailureHolder aFailure);
527 
528   // The last number of decoded output frames that we've reported to
529   // MediaDecoder::NotifyDecoded(). We diff the number of output video
530   // frames every time that DecodeVideoData() is called, and report the
531   // delta there.
532   uint64_t mLastReportedNumDecodedFrames;
533 
534   // Timestamp of the previous decoded keyframe, in microseconds.
535   int64_t mPreviousDecodedKeyframeTime_us;
536   // Default mLastDecodedKeyframeTime_us value, must be bigger than anything.
537   static const int64_t sNoPreviousDecodedKeyframe = INT64_MAX;
538 
539   RefPtr<layers::KnowsCompositor> mKnowsCompositor;
540 
541   // Metadata objects
542   // True if we've read the streams' metadata.
543   bool mInitDone;
544   MozPromiseHolder<MetadataPromise> mMetadataPromise;
545   bool IsEncrypted() const;
546 
547   // Set to true if any of our track buffers may be blocking.
548   bool mTrackDemuxersMayBlock;
549 
550   // Set the demuxed-only flag.
551   Atomic<bool> mDemuxOnly;
552 
553   // Seeking objects.
554   void SetSeekTarget(const SeekTarget& aTarget);
555   media::TimeUnit DemuxStartTime();
IsSeeking()556   bool IsSeeking() const { return mPendingSeekTime.isSome(); }
IsVideoSeeking()557   bool IsVideoSeeking() const
558   {
559     return IsSeeking() && mOriginalSeekTarget.IsVideoOnly();
560   }
561   void ScheduleSeek();
562   void AttemptSeek();
563   void OnSeekFailed(TrackType aTrack, const MediaResult& aError);
564   void DoVideoSeek();
565   void OnVideoSeekCompleted(media::TimeUnit aTime);
566   void OnVideoSeekFailed(const MediaResult& aError);
567   bool mSeekScheduled;
568 
NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)569   void NotifyCompositorUpdated(RefPtr<layers::KnowsCompositor> aKnowsCompositor)
570   {
571     mKnowsCompositor = aKnowsCompositor;
572   }
573 
574   void DoAudioSeek();
575   void OnAudioSeekCompleted(media::TimeUnit aTime);
576   void OnAudioSeekFailed(const MediaResult& aError);
577   // The SeekTarget that was last given to Seek()
578   SeekTarget mOriginalSeekTarget;
579   // Temporary seek information while we wait for the data
580   Maybe<media::TimeUnit> mFallbackSeekTime;
581   Maybe<media::TimeUnit> mPendingSeekTime;
582   MozPromiseHolder<SeekPromise> mSeekPromise;
583 
584   RefPtr<VideoFrameContainer> mVideoFrameContainer;
585   layers::ImageContainer* GetImageContainer();
586 
587   RefPtr<CDMProxy> mCDMProxy;
588 
589   RefPtr<GMPCrashHelper> mCrashHelper;
590 
591   void SetBlankDecode(TrackType aTrack, bool aIsBlankDecode);
592 
593   class DecoderFactory;
594   UniquePtr<DecoderFactory> mDecoderFactory;
595 
596   MediaEventListener mCompositorUpdatedListener;
597 };
598 
599 } // namespace mozilla
600 
601 #endif
602