1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef MOZILLA_TRACKBUFFERSMANAGER_H_ 8 #define MOZILLA_TRACKBUFFERSMANAGER_H_ 9 10 #include "mozilla/Atomics.h" 11 #include "mozilla/Maybe.h" 12 #include "mozilla/Mutex.h" 13 #include "mozilla/NotNull.h" 14 #include "mozilla/TaskQueue.h" 15 #include "mozilla/dom/MediaDebugInfoBinding.h" 16 17 #include "MediaContainerType.h" 18 #include "MediaData.h" 19 #include "MediaDataDemuxer.h" 20 #include "MediaResult.h" 21 #include "MediaSourceDecoder.h" 22 #include "MediaSpan.h" 23 #include "SourceBufferTask.h" 24 #include "TimeUnits.h" 25 #include "nsTArray.h" 26 27 namespace mozilla { 28 29 class AbstractThread; 30 class ContainerParser; 31 class MediaByteBuffer; 32 class MediaRawData; 33 class MediaSourceDemuxer; 34 class SourceBufferResource; 35 36 class SourceBufferTaskQueue { 37 public: 38 SourceBufferTaskQueue() = default; 39 ~SourceBufferTaskQueue()40 ~SourceBufferTaskQueue() { 41 MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed"); 42 } 43 Push(SourceBufferTask * aTask)44 void Push(SourceBufferTask* aTask) { mQueue.AppendElement(aTask); } 45 Pop()46 already_AddRefed<SourceBufferTask> Pop() { 47 if (!mQueue.Length()) { 48 return nullptr; 49 } 50 RefPtr<SourceBufferTask> task = std::move(mQueue[0]); 51 mQueue.RemoveElementAt(0); 52 return task.forget(); 53 } 54 Length()55 nsTArray<RefPtr<SourceBufferTask>>::size_type Length() const { 56 return mQueue.Length(); 57 } 58 59 private: 60 nsTArray<RefPtr<SourceBufferTask>> mQueue; 61 }; 62 63 DDLoggedTypeDeclName(TrackBuffersManager); 64 65 class TrackBuffersManager final 66 : public DecoderDoctorLifeLogger<TrackBuffersManager> { 67 public: 68 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager); 69 70 enum class EvictDataResult : int8_t { 71 NO_DATA_EVICTED, 72 CANT_EVICT, 73 BUFFER_FULL, 74 }; 75 76 typedef TrackInfo::TrackType TrackType; 77 typedef MediaData::Type MediaType; 78 typedef nsTArray<RefPtr<MediaRawData>> TrackBuffer; 79 typedef SourceBufferTask::AppendPromise AppendPromise; 80 typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise; 81 82 // Interface for SourceBuffer 83 TrackBuffersManager(MediaSourceDecoder* aParentDecoder, 84 const MediaContainerType& aType); 85 86 // Queue a task to add data to the end of the input buffer and run the MSE 87 // Buffer Append Algorithm 88 // 3.5.5 Buffer Append Algorithm. 89 // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append 90 RefPtr<AppendPromise> AppendData(already_AddRefed<MediaByteBuffer> aData, 91 const SourceBufferAttributes& aAttributes); 92 93 // Queue a task to abort any pending AppendData. 94 // Does nothing at this stage. 95 void AbortAppendData(); 96 97 // Queue a task to run MSE Reset Parser State Algorithm. 98 // 3.5.2 Reset Parser State 99 void ResetParserState(SourceBufferAttributes& aAttributes); 100 101 // Queue a task to run the MSE range removal algorithm. 102 // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal 103 RefPtr<RangeRemovalPromise> RangeRemoval(media::TimeUnit aStart, 104 media::TimeUnit aEnd); 105 106 // Schedule data eviction if necessary as the next call to AppendData will 107 // add aSize bytes. 108 // Eviction is done in two steps, first remove data up to aPlaybackTime 109 // and if still more space is needed remove from the end. 110 EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, 111 int64_t aSize); 112 113 // Queue a task to run ChangeType 114 void ChangeType(const MediaContainerType& aType); 115 116 // Returns the buffered range currently managed. 117 // This may be called on any thread. 118 // Buffered must conform to 119 // http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered 120 media::TimeIntervals Buffered() const; 121 media::TimeUnit HighestStartTime() const; 122 media::TimeUnit HighestEndTime() const; 123 124 // Return the size of the data managed by this SourceBufferContentManager. 125 int64_t GetSize() const; 126 127 // Indicate that the MediaSource parent object got into "ended" state. 128 void Ended(); 129 130 // The parent SourceBuffer is about to be destroyed. 131 void Detach(); 132 133 int64_t EvictionThreshold() const; 134 135 // Interface for MediaSourceDemuxer 136 MediaInfo GetMetadata() const; 137 const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const; 138 const media::TimeIntervals& Buffered(TrackInfo::TrackType) const; 139 const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const; 140 media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const; IsEnded()141 bool IsEnded() const { return mEnded; } 142 uint32_t Evictable(TrackInfo::TrackType aTrack) const; 143 media::TimeUnit Seek(TrackInfo::TrackType aTrack, 144 const media::TimeUnit& aTime, 145 const media::TimeUnit& aFuzz); 146 uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack, 147 const media::TimeUnit& aTimeThreadshold, 148 const media::TimeUnit& aFuzz, 149 bool& aFound); 150 151 already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack, 152 const media::TimeUnit& aFuzz, 153 MediaResult& aResult); 154 int32_t FindCurrentPosition(TrackInfo::TrackType aTrack, 155 const media::TimeUnit& aFuzz) const; 156 157 // Will set the next GetSample index if needed. This information is determined 158 // through the value of mNextSampleTimecode. Return false if the index 159 // couldn't be determined or if there's nothing more that could be demuxed. 160 // This occurs if either the track buffer doesn't contain the required 161 // timecode or is empty. 162 nsresult SetNextGetSampleIndexIfNeeded(TrackInfo::TrackType aTrack, 163 const media::TimeUnit& aFuzz); 164 165 media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack, 166 const media::TimeUnit& aFuzz); 167 168 void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; 169 void GetDebugInfo(dom::TrackBuffersManagerDebugInfo& aInfo); 170 171 private: 172 typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true> 173 CodedFrameProcessingPromise; 174 175 ~TrackBuffersManager(); 176 // All following functions run on the taskqueue. 177 RefPtr<AppendPromise> DoAppendData(already_AddRefed<MediaByteBuffer> aData, 178 const SourceBufferAttributes& aAttributes); 179 void ScheduleSegmentParserLoop(); 180 void SegmentParserLoop(); 181 void InitializationSegmentReceived(); 182 void ShutdownDemuxers(); 183 void CreateDemuxerforMIMEType(); 184 void ResetDemuxingState(); 185 void NeedMoreData(); 186 void RejectAppend(const MediaResult& aRejectValue, const char* aName); 187 // Will return a promise that will be resolved once all frames of the current 188 // media segment have been processed. 189 RefPtr<CodedFrameProcessingPromise> CodedFrameProcessing(); 190 void CompleteCodedFrameProcessing(); 191 // Called by ResetParserState. 192 void CompleteResetParserState(); 193 RefPtr<RangeRemovalPromise> CodedFrameRemovalWithPromise( 194 media::TimeInterval aInterval); 195 bool CodedFrameRemoval(media::TimeInterval aInterval); 196 // Removes all coded frames -- this is not to spec and should be used as a 197 // last resort to clear buffers only if other methods cannot. 198 void RemoveAllCodedFrames(); 199 void SetAppendState(SourceBufferAttributes::AppendState aAppendState); 200 HasVideo()201 bool HasVideo() const { return mVideoTracks.mNumTracks > 0; } HasAudio()202 bool HasAudio() const { return mAudioTracks.mNumTracks > 0; } 203 204 // The input buffer as per 205 // http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer 206 Maybe<MediaSpan> mInputBuffer; 207 // Buffer full flag as per 208 // https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag. Accessed 209 // on both the main thread and the task queue. 210 Atomic<bool> mBufferFull; 211 bool mFirstInitializationSegmentReceived; 212 bool mChangeTypeReceived; 213 // Set to true once a new segment is started. 214 bool mNewMediaSegmentStarted; 215 bool mActiveTrack; 216 MediaContainerType mType; 217 218 // ContainerParser objects and methods. 219 // Those are used to parse the incoming input buffer. 220 221 // Recreate the ContainerParser and if aReuseInitData is true then 222 // feed it with the previous init segment found. 223 void RecreateParser(bool aReuseInitData); 224 UniquePtr<ContainerParser> mParser; 225 226 // Demuxer objects and methods. 227 void AppendDataToCurrentInputBuffer(const MediaSpan& aData); 228 229 RefPtr<MediaByteBuffer> mInitData; 230 231 // Checks if a new set of init data is a repeat of the last set of init data 232 // received. Because streams may retransmit the same init data (or 233 // functionally equivalent init data) we do not want to perform costly 234 // operations each time we receive init data, only when it's actually 235 // different data. 236 bool IsRepeatInitData(const MediaInfo& aNewMediaInfo) const; 237 238 // Temporary input buffer to handle partial media segment header. 239 // We store the current input buffer content into it should we need to 240 // reinitialize the demuxer once we have some samples and a discontinuity is 241 // detected. 242 Maybe<MediaSpan> mPendingInputBuffer; 243 RefPtr<SourceBufferResource> mCurrentInputBuffer; 244 RefPtr<MediaDataDemuxer> mInputDemuxer; 245 // Length already processed in current media segment. 246 uint64_t mProcessedInput; 247 Maybe<media::TimeUnit> mLastParsedEndTime; 248 249 void OnDemuxerInitDone(const MediaResult& aResult); 250 void OnDemuxerInitFailed(const MediaResult& aFailure); 251 void OnDemuxerResetDone(const MediaResult& aResult); 252 MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest; 253 254 void OnDemuxFailed(TrackType aTrack, const MediaResult& aError); 255 void DoDemuxVideo(); 256 void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnVideoDemuxFailed(const MediaResult & aError)257 void OnVideoDemuxFailed(const MediaResult& aError) { 258 mVideoTracks.mDemuxRequest.Complete(); 259 OnDemuxFailed(TrackType::kVideoTrack, aError); 260 } 261 void DoDemuxAudio(); 262 void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples); OnAudioDemuxFailed(const MediaResult & aError)263 void OnAudioDemuxFailed(const MediaResult& aError) { 264 mAudioTracks.mDemuxRequest.Complete(); 265 OnDemuxFailed(TrackType::kAudioTrack, aError); 266 } 267 268 // Dispatches an "encrypted" event is any sample in array has initData 269 // present. 270 void MaybeDispatchEncryptedEvent( 271 const nsTArray<RefPtr<MediaRawData>>& aSamples); 272 273 void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict); 274 275 struct TrackData { TrackDataTrackData276 TrackData() : mNumTracks(0), mNeedRandomAccessPoint(true), mSizeBuffer(0) {} 277 Atomic<uint32_t> mNumTracks; 278 // Definition of variables: 279 // https://w3c.github.io/media-source/#track-buffers 280 // Last decode timestamp variable that stores the decode timestamp of the 281 // last coded frame appended in the current coded frame group. 282 // The variable is initially unset to indicate that no coded frames have 283 // been appended yet. 284 Maybe<media::TimeUnit> mLastDecodeTimestamp; 285 // Last frame duration variable that stores the coded frame duration of the 286 // last coded frame appended in the current coded frame group. 287 // The variable is initially unset to indicate that no coded frames have 288 // been appended yet. 289 Maybe<media::TimeUnit> mLastFrameDuration; 290 // Highest end timestamp variable that stores the highest coded frame end 291 // timestamp across all coded frames in the current coded frame group that 292 // were appended to this track buffer. 293 // The variable is initially unset to indicate that no coded frames have 294 // been appended yet. 295 Maybe<media::TimeUnit> mHighestEndTimestamp; 296 // Highest presentation timestamp in track buffer. 297 // Protected by global monitor, except when reading on the task queue as it 298 // is only written there. 299 media::TimeUnit mHighestStartTimestamp; 300 // Longest frame duration seen since last random access point. 301 // Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are 302 // set. 303 media::TimeUnit mLongestFrameDuration; 304 // Need random access point flag variable that keeps track of whether the 305 // track buffer is waiting for a random access point coded frame. 306 // The variable is initially set to true to indicate that random access 307 // point coded frame is needed before anything can be added to the track 308 // buffer. 309 bool mNeedRandomAccessPoint; 310 RefPtr<MediaTrackDemuxer> mDemuxer; 311 MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest; 312 // Highest end timestamp of the last media segment demuxed. 313 media::TimeUnit mLastParsedEndTime; 314 315 // If set, position where the next contiguous frame will be inserted. 316 // If a discontinuity is detected, it will be unset and recalculated upon 317 // the next insertion. 318 Maybe<uint32_t> mNextInsertionIndex; 319 // Samples just demuxed, but not yet parsed. 320 TrackBuffer mQueuedSamples; GetTrackBufferTrackData321 const TrackBuffer& GetTrackBuffer() const { 322 MOZ_RELEASE_ASSERT(mBuffers.Length(), 323 "TrackBuffer must have been created"); 324 return mBuffers.LastElement(); 325 } GetTrackBufferTrackData326 TrackBuffer& GetTrackBuffer() { 327 MOZ_RELEASE_ASSERT(mBuffers.Length(), 328 "TrackBuffer must have been created"); 329 return mBuffers.LastElement(); 330 } 331 // We only manage a single track of each type at this time. 332 nsTArray<TrackBuffer> mBuffers; 333 // Track buffer ranges variable that represents the presentation time ranges 334 // occupied by the coded frames currently stored in the track buffer. 335 media::TimeIntervals mBufferedRanges; 336 // Sanitized mBufferedRanges with a fuzz of half a sample's duration applied 337 // This buffered ranges is the basis of what is exposed to the JS. 338 media::TimeIntervals mSanitizedBufferedRanges; 339 // Byte size of all samples contained in this track buffer. 340 uint32_t mSizeBuffer; 341 // TrackInfo of the first metadata received. 342 RefPtr<TrackInfoSharedPtr> mInfo; 343 // TrackInfo of the last metadata parsed (updated with each init segment. 344 RefPtr<TrackInfoSharedPtr> mLastInfo; 345 346 // If set, position of the next sample to be retrieved by GetSample(). 347 // If the position is equal to the TrackBuffer's length, it indicates that 348 // we've reached EOS. 349 Maybe<uint32_t> mNextGetSampleIndex; 350 // Approximation of the next sample's decode timestamp. 351 media::TimeUnit mNextSampleTimecode; 352 // Approximation of the next sample's presentation timestamp. 353 media::TimeUnit mNextSampleTime; 354 355 struct EvictionIndex { EvictionIndexTrackData::EvictionIndex356 EvictionIndex() { Reset(); } ResetTrackData::EvictionIndex357 void Reset() { 358 mEvictable = 0; 359 mLastIndex = 0; 360 } 361 uint32_t mEvictable; 362 uint32_t mLastIndex; 363 }; 364 // Size of data that can be safely evicted during the next eviction 365 // cycle. 366 // We consider as evictable all frames up to the last keyframe prior to 367 // mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume 368 // that we can't yet evict data. 369 // Protected by global monitor, except when reading on the task queue as it 370 // is only written there. 371 EvictionIndex mEvictionIndex; 372 ResetAppendStateTrackData373 void ResetAppendState() { 374 mLastDecodeTimestamp.reset(); 375 mLastFrameDuration.reset(); 376 mHighestEndTimestamp.reset(); 377 mNeedRandomAccessPoint = true; 378 mNextInsertionIndex.reset(); 379 } 380 ResetTrackData381 void Reset() { 382 ResetAppendState(); 383 mEvictionIndex.Reset(); 384 for (auto& buffer : mBuffers) { 385 buffer.Clear(); 386 } 387 mSizeBuffer = 0; 388 mNextGetSampleIndex.reset(); 389 mBufferedRanges.Clear(); 390 mSanitizedBufferedRanges.Clear(); 391 } 392 393 void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; 394 }; 395 396 void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime); 397 void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData); 398 media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const; 399 bool CheckNextInsertionIndex(TrackData& aTrackData, 400 const media::TimeUnit& aSampleTime); 401 void InsertFrames(TrackBuffer& aSamples, 402 const media::TimeIntervals& aIntervals, 403 TrackData& aTrackData); 404 void UpdateHighestTimestamp(TrackData& aTrackData, 405 const media::TimeUnit& aHighestTime); 406 // Remove all frames and their dependencies contained in aIntervals. 407 // Return the index at which frames were first removed or 0 if no frames 408 // removed. 409 enum class RemovalMode { 410 kRemoveFrame, 411 kTruncateFrame, 412 }; 413 uint32_t RemoveFrames(const media::TimeIntervals& aIntervals, 414 TrackData& aTrackData, uint32_t aStartIndex, 415 RemovalMode aMode); 416 // Recalculate track's evictable amount. 417 void ResetEvictionIndex(TrackData& aTrackData); 418 void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex); 419 // Find index of sample. Return a negative value if not found. 420 uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer, 421 const media::TimeInterval& aInterval); 422 const MediaRawData* GetSample(TrackInfo::TrackType aTrack, uint32_t aIndex, 423 const media::TimeUnit& aExpectedDts, 424 const media::TimeUnit& aExpectedPts, 425 const media::TimeUnit& aFuzz); 426 void UpdateBufferedRanges(); 427 void RejectProcessing(const MediaResult& aRejectValue, const char* aName); 428 void ResolveProcessing(bool aResolveValue, const char* aName); 429 MozPromiseRequestHolder<CodedFrameProcessingPromise> mProcessingRequest; 430 MozPromiseHolder<CodedFrameProcessingPromise> mProcessingPromise; 431 432 // Trackbuffers definition. 433 nsTArray<const TrackData*> GetTracksList() const; 434 nsTArray<TrackData*> GetTracksList(); GetTracksData(TrackType aTrack)435 TrackData& GetTracksData(TrackType aTrack) { 436 switch (aTrack) { 437 case TrackType::kVideoTrack: 438 return mVideoTracks; 439 case TrackType::kAudioTrack: 440 default: 441 return mAudioTracks; 442 } 443 } GetTracksData(TrackType aTrack)444 const TrackData& GetTracksData(TrackType aTrack) const { 445 switch (aTrack) { 446 case TrackType::kVideoTrack: 447 return mVideoTracks; 448 case TrackType::kAudioTrack: 449 default: 450 return mAudioTracks; 451 } 452 } 453 TrackData mVideoTracks; 454 TrackData mAudioTracks; 455 456 // TaskQueue methods and objects. GetTaskQueueSafe()457 RefPtr<TaskQueue> GetTaskQueueSafe() const { 458 MutexAutoLock mut(mMutex); 459 return mTaskQueue; 460 } TaskQueueFromTaskQueue()461 NotNull<AbstractThread*> TaskQueueFromTaskQueue() const { 462 #ifdef DEBUG 463 RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe(); 464 MOZ_ASSERT(taskQueue && taskQueue->IsCurrentThreadIn()); 465 #endif 466 return WrapNotNull(mTaskQueue.get()); 467 } OnTaskQueue()468 bool OnTaskQueue() const { 469 auto taskQueue = TaskQueueFromTaskQueue(); 470 return taskQueue->IsCurrentThreadIn(); 471 } ResetTaskQueue()472 void ResetTaskQueue() { 473 MutexAutoLock mut(mMutex); 474 mTaskQueue = nullptr; 475 } 476 477 // SourceBuffer Queues and running context. 478 SourceBufferTaskQueue mQueue; 479 void QueueTask(SourceBufferTask* aTask); 480 void ProcessTasks(); 481 // Set if the TrackBuffersManager is currently processing a task. 482 // At this stage, this task is always a AppendBufferTask. 483 RefPtr<SourceBufferTask> mCurrentTask; 484 // Current SourceBuffer state for ongoing task. 485 // Its content is returned to the SourceBuffer once the AppendBufferTask has 486 // completed. 487 UniquePtr<SourceBufferAttributes> mSourceBufferAttributes; 488 // The current sourcebuffer append window. It's content is equivalent to 489 // mSourceBufferAttributes.mAppendWindowStart/End 490 media::TimeInterval mAppendWindow; 491 492 // Strong references to external objects. 493 nsMainThreadPtrHandle<MediaSourceDecoder> mParentDecoder; 494 495 const RefPtr<AbstractThread> mAbstractMainThread; 496 497 // Return public highest end time across all aTracks. 498 // Monitor must be held. 499 media::TimeUnit HighestEndTime( 500 nsTArray<const media::TimeIntervals*>& aTracks) const; 501 502 // Set to true if mediasource state changed to ended. 503 Atomic<bool> mEnded; 504 505 // Global size of this source buffer content. 506 Atomic<int64_t> mSizeSourceBuffer; 507 const int64_t mVideoEvictionThreshold; 508 const int64_t mAudioEvictionThreshold; 509 enum class EvictionState { 510 NO_EVICTION_NEEDED, 511 EVICTION_NEEDED, 512 EVICTION_COMPLETED, 513 }; 514 Atomic<EvictionState> mEvictionState; 515 516 // Monitor to protect following objects accessed across multiple threads. 517 mutable Mutex mMutex; 518 // mTaskQueue is only ever written after construction on the task queue. 519 // As such, it can be accessed while on task queue without the need for the 520 // mutex. 521 RefPtr<TaskQueue> mTaskQueue; 522 // Stable audio and video track time ranges. 523 media::TimeIntervals mVideoBufferedRanges; 524 media::TimeIntervals mAudioBufferedRanges; 525 // MediaInfo of the first init segment read. 526 MediaInfo mInfo; 527 }; 528 529 } // namespace mozilla 530 531 #endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */ 532