1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef MOZILLA_TRACKBUFFERSMANAGER_H_ 8 #define MOZILLA_TRACKBUFFERSMANAGER_H_ 9 10 #include "mozilla/Atomics.h" 11 #include "mozilla/Maybe.h" 12 #include "mozilla/Mutex.h" 13 #include "mozilla/NotNull.h" 14 #include "mozilla/TaskQueue.h" 15 #include "mozilla/dom/MediaDebugInfoBinding.h" 16 17 #include "MediaContainerType.h" 18 #include "MediaData.h" 19 #include "MediaDataDemuxer.h" 20 #include "MediaResult.h" 21 #include "MediaSourceDecoder.h" 22 #include "MediaSpan.h" 23 #include "SourceBufferTask.h" 24 #include "TimeUnits.h" 25 #include "nsTArray.h" 26 27 namespace mozilla { 28 29 class AbstractThread; 30 class ContainerParser; 31 class MediaByteBuffer; 32 class MediaRawData; 33 class MediaSourceDemuxer; 34 class SourceBufferResource; 35 36 class SourceBufferTaskQueue { 37 public: 38 SourceBufferTaskQueue() = default; 39 ~SourceBufferTaskQueue()40 ~SourceBufferTaskQueue() { 41 MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed"); 42 } 43 Push(SourceBufferTask * aTask)44 void Push(SourceBufferTask* aTask) { mQueue.AppendElement(aTask); } 45 Pop()46 already_AddRefed<SourceBufferTask> Pop() { 47 if (!mQueue.Length()) { 48 return nullptr; 49 } 50 RefPtr<SourceBufferTask> task = std::move(mQueue[0]); 51 mQueue.RemoveElementAt(0); 52 return task.forget(); 53 } 54 Length()55 nsTArray<RefPtr<SourceBufferTask>>::size_type Length() const { 56 return mQueue.Length(); 57 } 58 59 private: 60 nsTArray<RefPtr<SourceBufferTask>> mQueue; 61 }; 62 63 DDLoggedTypeDeclName(TrackBuffersManager); 64 65 class TrackBuffersManager final 66 : public DecoderDoctorLifeLogger<TrackBuffersManager> { 67 public: 68 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager); 69 70 enum class EvictDataResult : int8_t { 71 NO_DATA_EVICTED, 72 CANT_EVICT, 73 BUFFER_FULL, 74 }; 75 76 typedef TrackInfo::TrackType TrackType; 77 typedef MediaData::Type MediaType; 78 typedef nsTArray<RefPtr<MediaRawData>> TrackBuffer; 79 typedef SourceBufferTask::AppendPromise AppendPromise; 80 typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise; 81 82 // Interface for SourceBuffer 83 TrackBuffersManager(MediaSourceDecoder* aParentDecoder, 84 const MediaContainerType& aType); 85 86 // Queue a task to add data to the end of the input buffer and run the MSE 87 // Buffer Append Algorithm 88 // 3.5.5 Buffer Append Algorithm. 89 // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append 90 RefPtr<AppendPromise> AppendData(already_AddRefed<MediaByteBuffer> aData, 91 const SourceBufferAttributes& aAttributes); 92 93 // Queue a task to abort any pending AppendData. 94 // Does nothing at this stage. 95 void AbortAppendData(); 96 97 // Queue a task to run MSE Reset Parser State Algorithm. 98 // 3.5.2 Reset Parser State 99 void ResetParserState(SourceBufferAttributes& aAttributes); 100 101 // Queue a task to run the MSE range removal algorithm. 102 // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal 103 RefPtr<RangeRemovalPromise> RangeRemoval(media::TimeUnit aStart, 104 media::TimeUnit aEnd); 105 106 // Schedule data eviction if necessary as the next call to AppendData will 107 // add aSize bytes. 108 // Eviction is done in two steps, first remove data up to aPlaybackTime 109 // and if still more space is needed remove from the end. 110 EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, 111 int64_t aSize); 112 113 // Queue a task to run ChangeType 114 void ChangeType(const MediaContainerType& aType); 115 116 // Returns the buffered range currently managed. 117 // This may be called on any thread. 118 // Buffered must conform to 119 // http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered 120 media::TimeIntervals Buffered() const; 121 media::TimeUnit HighestStartTime() const; 122 media::TimeUnit HighestEndTime() const; 123 124 // Return the size of the data managed by this SourceBufferContentManager. 125 int64_t GetSize() const; 126 127 // Indicate that the MediaSource parent object got into "ended" state. 128 void Ended(); 129 130 // The parent SourceBuffer is about to be destroyed. 131 void Detach(); 132 133 int64_t EvictionThreshold() const; 134 135 // Interface for MediaSourceDemuxer 136 MediaInfo GetMetadata() const; 137 const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const; 138 const media::TimeIntervals& Buffered(TrackInfo::TrackType) const; 139 const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const; 140 media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const; IsEnded()141 bool IsEnded() const { return mEnded; } 142 uint32_t Evictable(TrackInfo::TrackType aTrack) const; 143 media::TimeUnit Seek(TrackInfo::TrackType aTrack, 144 const media::TimeUnit& aTime, 145 const media::TimeUnit& aFuzz); 146 uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack, 147 const media::TimeUnit& aTimeThreadshold, 148 const media::TimeUnit& aFuzz, 149 bool& aFound); 150 151 already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack, 152 const media::TimeUnit& aFuzz, 153 MediaResult& aResult); 154 int32_t FindCurrentPosition(TrackInfo::TrackType aTrack, 155 const media::TimeUnit& aFuzz) const; 156 157 // Will set the next GetSample index if needed. This information is determined 158 // through the value of mNextSampleTimecode. Return false if the index 159 // couldn't be determined or if there's nothing more that could be demuxed. 160 // This occurs if either the track buffer doesn't contain the required 161 // timecode or is empty. 162 nsresult SetNextGetSampleIndexIfNeeded(TrackInfo::TrackType aTrack, 163 const media::TimeUnit& aFuzz); 164 165 media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack, 166 const media::TimeUnit& aFuzz); 167 168 void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; 169 void GetDebugInfo(dom::TrackBuffersManagerDebugInfo& aInfo); 170 171 private: 172 typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true> 173 CodedFrameProcessingPromise; 174 175 ~TrackBuffersManager(); 176 // All following functions run on the taskqueue. 177 RefPtr<AppendPromise> DoAppendData(already_AddRefed<MediaByteBuffer> aData, 178 const SourceBufferAttributes& aAttributes); 179 void ScheduleSegmentParserLoop(); 180 void SegmentParserLoop(); 181 void InitializationSegmentReceived(); 182 void ShutdownDemuxers(); 183 void CreateDemuxerforMIMEType(); 184 void ResetDemuxingState(); 185 void NeedMoreData(); 186 void RejectAppend(const MediaResult& aRejectValue, const char* aName); 187 // Will return a promise that will be resolved once all frames of the current 188 // media segment have been processed. 189 RefPtr<CodedFrameProcessingPromise> CodedFrameProcessing(); 190 void CompleteCodedFrameProcessing(); 191 // Called by ResetParserState. 192 void CompleteResetParserState(); 193 RefPtr<RangeRemovalPromise> CodedFrameRemovalWithPromise( 194 media::TimeInterval aInterval); 195 bool CodedFrameRemoval(media::TimeInterval aInterval); 196 // Removes all coded frames -- this is not to spec and should be used as a 197 // last resort to clear buffers only if other methods cannot. 198 void RemoveAllCodedFrames(); 199 void SetAppendState(SourceBufferAttributes::AppendState aAppendState); 200 HasVideo()201 bool HasVideo() const { return mVideoTracks.mNumTracks > 0; } HasAudio()202 bool HasAudio() const { return mAudioTracks.mNumTracks > 0; } 203 204 // The input buffer as per 205 // http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer 206 Maybe<MediaSpan> mInputBuffer; 207 // Buffer full flag as per 208 // https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag. Accessed 209 // on both the main thread and the task queue. 210 Atomic<bool> mBufferFull; 211 bool mFirstInitializationSegmentReceived; 212 bool mChangeTypeReceived; 213 // Set to true once a new segment is started. 214 bool mNewMediaSegmentStarted; 215 bool mActiveTrack; 216 MediaContainerType mType; 217 218 // ContainerParser objects and methods. 219 // Those are used to parse the incoming input buffer. 220 221 // Recreate the ContainerParser and if aReuseInitData is true then 222 // feed it with the previous init segment found. 223 void RecreateParser(bool aReuseInitData); 224 UniquePtr<ContainerParser> mParser; 225 226 // Demuxer objects and methods. 227 void AppendDataToCurrentInputBuffer(const MediaSpan& aData); 228 229 RefPtr<MediaByteBuffer> mInitData; 230 // Temporary input buffer to handle partial media segment header. 231 // We store the current input buffer content into it should we need to 232 // reinitialize the demuxer once we have some samples and a discontinuity is 233 // detected. 234 Maybe<MediaSpan> mPendingInputBuffer; 235 RefPtr<SourceBufferResource> mCurrentInputBuffer; 236 RefPtr<MediaDataDemuxer> mInputDemuxer; 237 // Length already processed in current media segment. 238 uint64_t mProcessedInput; 239 Maybe<media::TimeUnit> mLastParsedEndTime; 240 241 void OnDemuxerInitDone(const MediaResult& aResult); 242 void OnDemuxerInitFailed(const MediaResult& aFailure); 243 void OnDemuxerResetDone(const MediaResult& aResult); 244 MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; 245 246 void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); 247 void DoDemuxVideo(); 248 void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnVideoDemuxFailed(const MediaResult & aError)249 void OnVideoDemuxFailed(const MediaResult& aError) { 250 mVideoTracks.mDemuxRequest.Complete(); 251 OnDemuxFailed(TrackType::kVideoTrack, aError); 252 } 253 void DoDemuxAudio(); 254 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnAudioDemuxFailed(const MediaResult & aError)255 void OnAudioDemuxFailed(const MediaResult& aError) { 256 mAudioTracks.mDemuxRequest.Complete(); 257 OnDemuxFailed(TrackType::kAudioTrack, aError); 258 } 259 260 // Dispatches an "encrypted" event is any sample in array has initData 261 // present. 262 void MaybeDispatchEncryptedEvent( 263 const nsTArray<RefPtr<MediaRawData>>& aSamples); 264 265 void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict); 266 267 struct TrackData { TrackDataTrackData268 TrackData() : mNumTracks(0), mNeedRandomAccessPoint(true), mSizeBuffer(0) {} 269 uint32_t mNumTracks; 270 // Definition of variables: 271 // https://w3c.github.io/media-source/#track-buffers 272 // Last decode timestamp variable that stores the decode timestamp of the 273 // last coded frame appended in the current coded frame group. 274 // The variable is initially unset to indicate that no coded frames have 275 // been appended yet. 276 Maybe<media::TimeUnit> mLastDecodeTimestamp; 277 // Last frame duration variable that stores the coded frame duration of the 278 // last coded frame appended in the current coded frame group. 279 // The variable is initially unset to indicate that no coded frames have 280 // been appended yet. 281 Maybe<media::TimeUnit> mLastFrameDuration; 282 // Highest end timestamp variable that stores the highest coded frame end 283 // timestamp across all coded frames in the current coded frame group that 284 // were appended to this track buffer. 285 // The variable is initially unset to indicate that no coded frames have 286 // been appended yet. 287 Maybe<media::TimeUnit> mHighestEndTimestamp; 288 // Highest presentation timestamp in track buffer. 289 // Protected by global monitor, except when reading on the task queue as it 290 // is only written there. 291 media::TimeUnit mHighestStartTimestamp; 292 // Longest frame duration seen since last random access point. 293 // Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are 294 // set. 295 media::TimeUnit mLongestFrameDuration; 296 // Need random access point flag variable that keeps track of whether the 297 // track buffer is waiting for a random access point coded frame. 298 // The variable is initially set to true to indicate that random access 299 // point coded frame is needed before anything can be added to the track 300 // buffer. 301 bool mNeedRandomAccessPoint; 302 RefPtr<MediaTrackDemuxer> mDemuxer; 303 MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; 304 // Highest end timestamp of the last media segment demuxed. 305 media::TimeUnit mLastParsedEndTime; 306 307 // If set, position where the next contiguous frame will be inserted. 308 // If a discontinuity is detected, it will be unset and recalculated upon 309 // the next insertion. 310 Maybe<uint32_t> mNextInsertionIndex; 311 // Samples just demuxed, but not yet parsed. 312 TrackBuffer mQueuedSamples; GetTrackBufferTrackData313 const TrackBuffer& GetTrackBuffer() const { 314 MOZ_RELEASE_ASSERT(mBuffers.Length(), 315 "TrackBuffer must have been created"); 316 return mBuffers.LastElement(); 317 } GetTrackBufferTrackData318 TrackBuffer& GetTrackBuffer() { 319 MOZ_RELEASE_ASSERT(mBuffers.Length(), 320 "TrackBuffer must have been created"); 321 return mBuffers.LastElement(); 322 } 323 // We only manage a single track of each type at this time. 324 nsTArray<TrackBuffer> mBuffers; 325 // Track buffer ranges variable that represents the presentation time ranges 326 // occupied by the coded frames currently stored in the track buffer. 327 media::TimeIntervals mBufferedRanges; 328 // Sanitized mBufferedRanges with a fuzz of half a sample's duration applied 329 // This buffered ranges is the basis of what is exposed to the JS. 330 media::TimeIntervals mSanitizedBufferedRanges; 331 // Byte size of all samples contained in this track buffer. 332 uint32_t mSizeBuffer; 333 // TrackInfo of the first metadata received. 334 RefPtr<TrackInfoSharedPtr> mInfo; 335 // TrackInfo of the last metadata parsed (updated with each init segment. 336 RefPtr<TrackInfoSharedPtr> mLastInfo; 337 338 // If set, position of the next sample to be retrieved by GetSample(). 339 // If the position is equal to the TrackBuffer's length, it indicates that 340 // we've reached EOS. 341 Maybe<uint32_t> mNextGetSampleIndex; 342 // Approximation of the next sample's decode timestamp. 343 media::TimeUnit mNextSampleTimecode; 344 // Approximation of the next sample's presentation timestamp. 345 media::TimeUnit mNextSampleTime; 346 347 struct EvictionIndex { EvictionIndexTrackData::EvictionIndex348 EvictionIndex() { Reset(); } ResetTrackData::EvictionIndex349 void Reset() { 350 mEvictable = 0; 351 mLastIndex = 0; 352 } 353 uint32_t mEvictable; 354 uint32_t mLastIndex; 355 }; 356 // Size of data that can be safely evicted during the next eviction 357 // cycle. 358 // We consider as evictable all frames up to the last keyframe prior to 359 // mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume 360 // that we can't yet evict data. 361 // Protected by global monitor, except when reading on the task queue as it 362 // is only written there. 363 EvictionIndex mEvictionIndex; 364 ResetAppendStateTrackData365 void ResetAppendState() { 366 mLastDecodeTimestamp.reset(); 367 mLastFrameDuration.reset(); 368 mHighestEndTimestamp.reset(); 369 mNeedRandomAccessPoint = true; 370 mNextInsertionIndex.reset(); 371 } 372 ResetTrackData373 void Reset() { 374 ResetAppendState(); 375 mEvictionIndex.Reset(); 376 for (auto& buffer : mBuffers) { 377 buffer.Clear(); 378 } 379 mSizeBuffer = 0; 380 mNextGetSampleIndex.reset(); 381 mBufferedRanges.Clear(); 382 mSanitizedBufferedRanges.Clear(); 383 } 384 385 void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; 386 }; 387 388 void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime); 389 void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData); 390 media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const; 391 bool CheckNextInsertionIndex(TrackData& aTrackData, 392 const media::TimeUnit& aSampleTime); 393 void InsertFrames(TrackBuffer& aSamples, 394 const media::TimeIntervals& aIntervals, 395 TrackData& aTrackData); 396 void UpdateHighestTimestamp(TrackData& aTrackData, 397 const media::TimeUnit& aHighestTime); 398 // Remove all frames and their dependencies contained in aIntervals. 399 // Return the index at which frames were first removed or 0 if no frames 400 // removed. 401 enum class RemovalMode { 402 kRemoveFrame, 403 kTruncateFrame, 404 }; 405 uint32_t RemoveFrames(const media::TimeIntervals& aIntervals, 406 TrackData& aTrackData, uint32_t aStartIndex, 407 RemovalMode aMode); 408 // Recalculate track's evictable amount. 409 void ResetEvictionIndex(TrackData& aTrackData); 410 void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex); 411 // Find index of sample. Return a negative value if not found. 412 uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer, 413 const media::TimeInterval& aInterval); 414 const MediaRawData* GetSample(TrackInfo::TrackType aTrack, uint32_t aIndex, 415 const media::TimeUnit& aExpectedDts, 416 const media::TimeUnit& aExpectedPts, 417 const media::TimeUnit& aFuzz); 418 void UpdateBufferedRanges(); 419 void RejectProcessing(const MediaResult& aRejectValue, const char* aName); 420 void ResolveProcessing(bool aResolveValue, const char* aName); 421 MozPromiseRequestHolder<CodedFrameProcessingPromise> mProcessingRequest; 422 MozPromiseHolder<CodedFrameProcessingPromise> mProcessingPromise; 423 424 // Trackbuffers definition. 425 nsTArray<const TrackData*> GetTracksList() const; 426 nsTArray<TrackData*> GetTracksList(); GetTracksData(TrackType aTrack)427 TrackData& GetTracksData(TrackType aTrack) { 428 switch (aTrack) { 429 case TrackType::kVideoTrack: 430 return mVideoTracks; 431 case TrackType::kAudioTrack: 432 default: 433 return mAudioTracks; 434 } 435 } GetTracksData(TrackType aTrack)436 const TrackData& GetTracksData(TrackType aTrack) const { 437 switch (aTrack) { 438 case TrackType::kVideoTrack: 439 return mVideoTracks; 440 case TrackType::kAudioTrack: 441 default: 442 return mAudioTracks; 443 } 444 } 445 TrackData mVideoTracks; 446 TrackData mAudioTracks; 447 448 // TaskQueue methods and objects. GetTaskQueueSafe()449 RefPtr<TaskQueue> GetTaskQueueSafe() const { 450 MutexAutoLock mut(mMutex); 451 return mTaskQueue; 452 } TaskQueueFromTaskQueue()453 NotNull<AbstractThread*> TaskQueueFromTaskQueue() const { 454 #ifdef DEBUG 455 RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe(); 456 MOZ_ASSERT(taskQueue && taskQueue->IsCurrentThreadIn()); 457 #endif 458 return WrapNotNull(mTaskQueue.get()); 459 } OnTaskQueue()460 bool OnTaskQueue() const { 461 auto taskQueue = TaskQueueFromTaskQueue(); 462 return taskQueue->IsCurrentThreadIn(); 463 } ResetTaskQueue()464 void ResetTaskQueue() { 465 MutexAutoLock mut(mMutex); 466 mTaskQueue = nullptr; 467 } 468 469 // SourceBuffer Queues and running context. 470 SourceBufferTaskQueue mQueue; 471 void QueueTask(SourceBufferTask* aTask); 472 void ProcessTasks(); 473 // Set if the TrackBuffersManager is currently processing a task. 474 // At this stage, this task is always a AppendBufferTask. 475 RefPtr<SourceBufferTask> mCurrentTask; 476 // Current SourceBuffer state for ongoing task. 477 // Its content is returned to the SourceBuffer once the AppendBufferTask has 478 // completed. 479 UniquePtr<SourceBufferAttributes> mSourceBufferAttributes; 480 // The current sourcebuffer append window. It's content is equivalent to 481 // mSourceBufferAttributes.mAppendWindowStart/End 482 media::TimeInterval mAppendWindow; 483 484 // Strong references to external objects. 485 nsMainThreadPtrHandle<MediaSourceDecoder> mParentDecoder; 486 487 const RefPtr<AbstractThread> mAbstractMainThread; 488 489 // Return public highest end time across all aTracks. 490 // Monitor must be held. 491 media::TimeUnit HighestEndTime( 492 nsTArray<const media::TimeIntervals*>& aTracks) const; 493 494 // Set to true if mediasource state changed to ended. 495 Atomic<bool> mEnded; 496 497 // Global size of this source buffer content. 498 Atomic<int64_t> mSizeSourceBuffer; 499 const int64_t mVideoEvictionThreshold; 500 const int64_t mAudioEvictionThreshold; 501 enum class EvictionState { 502 NO_EVICTION_NEEDED, 503 EVICTION_NEEDED, 504 EVICTION_COMPLETED, 505 }; 506 Atomic<EvictionState> mEvictionState; 507 508 // Monitor to protect following objects accessed across multiple threads. 509 mutable Mutex mMutex; 510 // mTaskQueue is only ever written after construction on the task queue. 511 // As such, it can be accessed while on task queue without the need for the 512 // mutex. 513 RefPtr<TaskQueue> mTaskQueue; 514 // Stable audio and video track time ranges. 515 media::TimeIntervals mVideoBufferedRanges; 516 media::TimeIntervals mAudioBufferedRanges; 517 // MediaInfo of the first init segment read. 518 MediaInfo mInfo; 519 }; 520 521 } // namespace mozilla 522 523 #endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */ 524