1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaSourceDemuxer.h"
8
9 #include "MediaSourceUtils.h"
10 #include "OpusDecoder.h"
11 #include "SourceBufferList.h"
12 #include "VorbisDecoder.h"
13 #include "VideoUtils.h"
14 #include "nsPrintfCString.h"
15
16 #include <algorithm>
17 #include <limits>
18 #include <stdint.h>
19
20 namespace mozilla {
21
22 typedef TrackInfo::TrackType TrackType;
23 using media::TimeIntervals;
24 using media::TimeUnit;
25
MediaSourceDemuxer(AbstractThread * aAbstractMainThread)26 MediaSourceDemuxer::MediaSourceDemuxer(AbstractThread* aAbstractMainThread)
27 : mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
28 "MediaSourceDemuxer::mTaskQueue")),
29 mMonitor("MediaSourceDemuxer") {
30 MOZ_ASSERT(NS_IsMainThread());
31 }
32
33 constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ;
34
Init()35 RefPtr<MediaSourceDemuxer::InitPromise> MediaSourceDemuxer::Init() {
36 RefPtr<MediaSourceDemuxer> self = this;
37 return InvokeAsync(GetTaskQueue(), __func__, [self]() {
38 if (self->ScanSourceBuffersForContent()) {
39 return InitPromise::CreateAndResolve(NS_OK, __func__);
40 }
41
42 RefPtr<InitPromise> p = self->mInitPromise.Ensure(__func__);
43
44 return p;
45 });
46 }
47
AddSizeOfResources(MediaSourceDecoder::ResourceSizes * aSizes)48 void MediaSourceDemuxer::AddSizeOfResources(
49 MediaSourceDecoder::ResourceSizes* aSizes) {
50 MOZ_ASSERT(NS_IsMainThread());
51
52 // NB: The track buffers must only be accessed on the TaskQueue.
53 RefPtr<MediaSourceDemuxer> self = this;
54 RefPtr<MediaSourceDecoder::ResourceSizes> sizes = aSizes;
55 nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
56 "MediaSourceDemuxer::AddSizeOfResources", [self, sizes]() {
57 for (const RefPtr<TrackBuffersManager>& manager :
58 self->mSourceBuffers) {
59 manager->AddSizeOfResources(sizes);
60 }
61 });
62
63 nsresult rv = GetTaskQueue()->Dispatch(task.forget());
64 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
65 Unused << rv;
66 }
67
NotifyInitDataArrived()68 void MediaSourceDemuxer::NotifyInitDataArrived() {
69 RefPtr<MediaSourceDemuxer> self = this;
70 nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
71 "MediaSourceDemuxer::NotifyInitDataArrived", [self]() {
72 if (self->mInitPromise.IsEmpty()) {
73 return;
74 }
75 if (self->ScanSourceBuffersForContent()) {
76 self->mInitPromise.ResolveIfExists(NS_OK, __func__);
77 }
78 });
79 nsresult rv = GetTaskQueue()->Dispatch(task.forget());
80 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
81 Unused << rv;
82 }
83
ScanSourceBuffersForContent()84 bool MediaSourceDemuxer::ScanSourceBuffersForContent() {
85 MOZ_ASSERT(OnTaskQueue());
86
87 if (mSourceBuffers.IsEmpty()) {
88 return false;
89 }
90
91 MonitorAutoLock mon(mMonitor);
92
93 bool haveEmptySourceBuffer = false;
94 for (const auto& sourceBuffer : mSourceBuffers) {
95 MediaInfo info = sourceBuffer->GetMetadata();
96 if (!info.HasAudio() && !info.HasVideo()) {
97 haveEmptySourceBuffer = true;
98 }
99 if (info.HasAudio() && !mAudioTrack) {
100 mInfo.mAudio = info.mAudio;
101 mAudioTrack = sourceBuffer;
102 }
103 if (info.HasVideo() && !mVideoTrack) {
104 mInfo.mVideo = info.mVideo;
105 mVideoTrack = sourceBuffer;
106 }
107 if (info.IsEncrypted() && !mInfo.IsEncrypted()) {
108 mInfo.mCrypto = info.mCrypto;
109 }
110 }
111 if (mInfo.HasAudio() && mInfo.HasVideo()) {
112 // We have both audio and video. We can ignore non-ready source buffer.
113 return true;
114 }
115 return !haveEmptySourceBuffer;
116 }
117
GetNumberTracks(TrackType aType) const118 uint32_t MediaSourceDemuxer::GetNumberTracks(TrackType aType) const {
119 MonitorAutoLock mon(mMonitor);
120
121 switch (aType) {
122 case TrackType::kAudioTrack:
123 return mInfo.HasAudio() ? 1u : 0;
124 case TrackType::kVideoTrack:
125 return mInfo.HasVideo() ? 1u : 0;
126 default:
127 return 0;
128 }
129 }
130
GetTrackDemuxer(TrackType aType,uint32_t aTrackNumber)131 already_AddRefed<MediaTrackDemuxer> MediaSourceDemuxer::GetTrackDemuxer(
132 TrackType aType, uint32_t aTrackNumber) {
133 RefPtr<TrackBuffersManager> manager = GetManager(aType);
134 if (!manager) {
135 return nullptr;
136 }
137 RefPtr<MediaSourceTrackDemuxer> e =
138 new MediaSourceTrackDemuxer(this, aType, manager);
139 DDLINKCHILD("track demuxer", e.get());
140 mDemuxers.AppendElement(e);
141 return e.forget();
142 }
143
IsSeekable() const144 bool MediaSourceDemuxer::IsSeekable() const { return true; }
145
GetCrypto()146 UniquePtr<EncryptionInfo> MediaSourceDemuxer::GetCrypto() {
147 MonitorAutoLock mon(mMonitor);
148 auto crypto = MakeUnique<EncryptionInfo>();
149 *crypto = mInfo.mCrypto;
150 return crypto;
151 }
152
AttachSourceBuffer(RefPtr<TrackBuffersManager> & aSourceBuffer)153 void MediaSourceDemuxer::AttachSourceBuffer(
154 RefPtr<TrackBuffersManager>& aSourceBuffer) {
155 nsCOMPtr<nsIRunnable> task = NewRunnableMethod<RefPtr<TrackBuffersManager>&&>(
156 "MediaSourceDemuxer::DoAttachSourceBuffer", this,
157 &MediaSourceDemuxer::DoAttachSourceBuffer, aSourceBuffer);
158 nsresult rv = GetTaskQueue()->Dispatch(task.forget());
159 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
160 Unused << rv;
161 }
162
DoAttachSourceBuffer(RefPtr<mozilla::TrackBuffersManager> && aSourceBuffer)163 void MediaSourceDemuxer::DoAttachSourceBuffer(
164 RefPtr<mozilla::TrackBuffersManager>&& aSourceBuffer) {
165 MOZ_ASSERT(OnTaskQueue());
166 mSourceBuffers.AppendElement(std::move(aSourceBuffer));
167 ScanSourceBuffersForContent();
168 }
169
DetachSourceBuffer(RefPtr<TrackBuffersManager> & aSourceBuffer)170 void MediaSourceDemuxer::DetachSourceBuffer(
171 RefPtr<TrackBuffersManager>& aSourceBuffer) {
172 nsCOMPtr<nsIRunnable> task = NewRunnableMethod<RefPtr<TrackBuffersManager>&&>(
173 "MediaSourceDemuxer::DoDetachSourceBuffer", this,
174 &MediaSourceDemuxer::DoDetachSourceBuffer, aSourceBuffer);
175 nsresult rv = GetTaskQueue()->Dispatch(task.forget());
176 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
177 Unused << rv;
178 }
179
DoDetachSourceBuffer(RefPtr<TrackBuffersManager> && aSourceBuffer)180 void MediaSourceDemuxer::DoDetachSourceBuffer(
181 RefPtr<TrackBuffersManager>&& aSourceBuffer) {
182 MOZ_ASSERT(OnTaskQueue());
183 mSourceBuffers.RemoveElementsBy(
184 [&aSourceBuffer](const RefPtr<TrackBuffersManager> aLinkedSourceBuffer) {
185 return aLinkedSourceBuffer == aSourceBuffer;
186 });
187 {
188 MonitorAutoLock mon(mMonitor);
189 if (aSourceBuffer == mAudioTrack) {
190 mAudioTrack = nullptr;
191 }
192 if (aSourceBuffer == mVideoTrack) {
193 mVideoTrack = nullptr;
194 }
195 }
196
197 for (auto& demuxer : mDemuxers) {
198 if (demuxer->HasManager(aSourceBuffer)) {
199 demuxer->DetachManager();
200 }
201 }
202 ScanSourceBuffersForContent();
203 }
204
GetTrackInfo(TrackType aTrack)205 TrackInfo* MediaSourceDemuxer::GetTrackInfo(TrackType aTrack) {
206 MonitorAutoLock mon(mMonitor);
207 switch (aTrack) {
208 case TrackType::kAudioTrack:
209 return &mInfo.mAudio;
210 case TrackType::kVideoTrack:
211 return &mInfo.mVideo;
212 default:
213 return nullptr;
214 }
215 }
216
GetManager(TrackType aTrack)217 RefPtr<TrackBuffersManager> MediaSourceDemuxer::GetManager(TrackType aTrack) {
218 MonitorAutoLock mon(mMonitor);
219 switch (aTrack) {
220 case TrackType::kAudioTrack:
221 return mAudioTrack;
222 case TrackType::kVideoTrack:
223 return mVideoTrack;
224 default:
225 return nullptr;
226 }
227 }
228
~MediaSourceDemuxer()229 MediaSourceDemuxer::~MediaSourceDemuxer() {
230 mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
231 }
232
GetDebugInfo(dom::MediaSourceDemuxerDebugInfo & aInfo)233 void MediaSourceDemuxer::GetDebugInfo(dom::MediaSourceDemuxerDebugInfo& aInfo) {
234 MonitorAutoLock mon(mMonitor);
235 if (mAudioTrack) {
236 mAudioTrack->GetDebugInfo(aInfo.mAudioTrack);
237 }
238 if (mVideoTrack) {
239 mVideoTrack->GetDebugInfo(aInfo.mVideoTrack);
240 }
241 }
242
MediaSourceTrackDemuxer(MediaSourceDemuxer * aParent,TrackInfo::TrackType aType,TrackBuffersManager * aManager)243 MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent,
244 TrackInfo::TrackType aType,
245 TrackBuffersManager* aManager)
246 : mParent(aParent),
247 mType(aType),
248 mMonitor("MediaSourceTrackDemuxer"),
249 mManager(aManager),
250 mReset(true),
251 mPreRoll(TimeUnit::FromMicroseconds(
252 OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) ||
253 VorbisDataDecoder::IsVorbis(
254 mParent->GetTrackInfo(mType)->mMimeType)
255 ? 80000
256 : mParent->GetTrackInfo(mType)->mMimeType.EqualsLiteral(
257 "audio/mp4a-latm")
258 // AAC encoder delay is by default 2112 audio frames.
259 // See
260 // https://developer.apple.com/library/content/documentation/QuickTime/QTFF/QTFFAppenG/QTFFAppenG.html
261 // So we always seek 2112 frames
262 ? (2112 * 1000000ULL /
263 mParent->GetTrackInfo(mType)->GetAsAudioInfo()->mRate)
264 : 0)) {}
265
GetInfo() const266 UniquePtr<TrackInfo> MediaSourceTrackDemuxer::GetInfo() const {
267 return mParent->GetTrackInfo(mType)->Clone();
268 }
269
Seek(const TimeUnit & aTime)270 RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::Seek(
271 const TimeUnit& aTime) {
272 MOZ_ASSERT(mParent, "Called after BreackCycle()");
273 return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
274 &MediaSourceTrackDemuxer::DoSeek, aTime);
275 }
276
277 RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
GetSamples(int32_t aNumSamples)278 MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples) {
279 MOZ_ASSERT(mParent, "Called after BreackCycle()");
280 return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
281 &MediaSourceTrackDemuxer::DoGetSamples, aNumSamples);
282 }
283
Reset()284 void MediaSourceTrackDemuxer::Reset() {
285 MOZ_ASSERT(mParent, "Called after BreackCycle()");
286 RefPtr<MediaSourceTrackDemuxer> self = this;
287 nsCOMPtr<nsIRunnable> task =
288 NS_NewRunnableFunction("MediaSourceTrackDemuxer::Reset", [self]() {
289 self->mNextSample.reset();
290 self->mReset = true;
291 if (!self->mManager) {
292 return;
293 }
294 MOZ_ASSERT(self->OnTaskQueue());
295 self->mManager->Seek(self->mType, TimeUnit::Zero(), TimeUnit::Zero());
296 {
297 MonitorAutoLock mon(self->mMonitor);
298 self->mNextRandomAccessPoint =
299 self->mManager->GetNextRandomAccessPoint(
300 self->mType, MediaSourceDemuxer::EOS_FUZZ);
301 }
302 });
303 nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
304 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
305 Unused << rv;
306 }
307
GetNextRandomAccessPoint(TimeUnit * aTime)308 nsresult MediaSourceTrackDemuxer::GetNextRandomAccessPoint(TimeUnit* aTime) {
309 MonitorAutoLock mon(mMonitor);
310 *aTime = mNextRandomAccessPoint;
311 return NS_OK;
312 }
313
314 RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
SkipToNextRandomAccessPoint(const TimeUnit & aTimeThreshold)315 MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint(
316 const TimeUnit& aTimeThreshold) {
317 return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
318 &MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint,
319 aTimeThreshold);
320 }
321
GetBuffered()322 media::TimeIntervals MediaSourceTrackDemuxer::GetBuffered() {
323 MonitorAutoLock mon(mMonitor);
324 if (!mManager) {
325 return media::TimeIntervals();
326 }
327 return mManager->Buffered();
328 }
329
BreakCycles()330 void MediaSourceTrackDemuxer::BreakCycles() {
331 RefPtr<MediaSourceTrackDemuxer> self = this;
332 nsCOMPtr<nsIRunnable> task =
333 NS_NewRunnableFunction("MediaSourceTrackDemuxer::BreakCycles", [self]() {
334 self->DetachManager();
335 self->mParent = nullptr;
336 });
337 nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
338 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
339 Unused << rv;
340 }
341
DoSeek(const TimeUnit & aTime)342 RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::DoSeek(
343 const TimeUnit& aTime) {
344 if (!mManager) {
345 return SeekPromise::CreateAndReject(
346 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
347 RESULT_DETAIL("manager is detached.")),
348 __func__);
349 }
350
351 MOZ_ASSERT(OnTaskQueue());
352 TimeIntervals buffered = mManager->Buffered(mType);
353 // Fuzz factor represents a +/- threshold. So when seeking it allows the gap
354 // to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap.
355 buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
356 TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::Zero());
357
358 if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) {
359 // We're attempting to seek past the end time. Cap seekTime so that we seek
360 // to the last sample instead.
361 seekTime = std::max(mManager->HighestStartTime(mType) - mPreRoll,
362 TimeUnit::Zero());
363 }
364 if (!buffered.ContainsWithStrictEnd(seekTime)) {
365 if (!buffered.ContainsWithStrictEnd(aTime)) {
366 // We don't have the data to seek to.
367 return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
368 __func__);
369 }
370 // Theoretically we should reject the promise with WAITING_FOR_DATA,
371 // however, to avoid unwanted regressions we assume that if at this time
372 // we don't have the wanted data it won't come later.
373 // Instead of using the pre-rolled time, use the earliest time available in
374 // the interval.
375 TimeIntervals::IndexType index = buffered.Find(aTime);
376 MOZ_ASSERT(index != TimeIntervals::NoIndex);
377 seekTime = buffered[index].mStart;
378 }
379 seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ);
380 MediaResult result = NS_OK;
381 RefPtr<MediaRawData> sample =
382 mManager->GetSample(mType, TimeUnit::Zero(), result);
383 MOZ_ASSERT(NS_SUCCEEDED(result) && sample);
384 mNextSample = Some(sample);
385 mReset = false;
386 {
387 MonitorAutoLock mon(mMonitor);
388 mNextRandomAccessPoint =
389 mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
390 }
391 return SeekPromise::CreateAndResolve(seekTime, __func__);
392 }
393
394 RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
DoGetSamples(int32_t aNumSamples)395 MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples) {
396 if (!mManager) {
397 return SamplesPromise::CreateAndReject(
398 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
399 RESULT_DETAIL("manager is detached.")),
400 __func__);
401 }
402
403 MOZ_ASSERT(OnTaskQueue());
404 if (mReset) {
405 // If a seek (or reset) was recently performed, we ensure that the data
406 // we are about to retrieve is still available.
407 TimeIntervals buffered = mManager->Buffered(mType);
408 buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
409
410 if (buffered.IsEmpty() && mManager->IsEnded()) {
411 return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
412 __func__);
413 }
414 if (!buffered.ContainsWithStrictEnd(TimeUnit::Zero())) {
415 return SamplesPromise::CreateAndReject(
416 NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
417 }
418 mReset = false;
419 }
420 RefPtr<MediaRawData> sample;
421 if (mNextSample) {
422 sample = mNextSample.ref();
423 mNextSample.reset();
424 } else {
425 MediaResult result = NS_OK;
426 sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result);
427 if (!sample) {
428 if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM ||
429 result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
430 return SamplesPromise::CreateAndReject(
431 (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded())
432 ? NS_ERROR_DOM_MEDIA_END_OF_STREAM
433 : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
434 __func__);
435 }
436 return SamplesPromise::CreateAndReject(result, __func__);
437 }
438 }
439 RefPtr<SamplesHolder> samples = new SamplesHolder;
440 samples->AppendSample(sample);
441 if (mNextRandomAccessPoint <= sample->mTime) {
442 MonitorAutoLock mon(mMonitor);
443 mNextRandomAccessPoint =
444 mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
445 }
446 return SamplesPromise::CreateAndResolve(samples, __func__);
447 }
448
449 RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
DoSkipToNextRandomAccessPoint(const TimeUnit & aTimeThreadshold)450 MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(
451 const TimeUnit& aTimeThreadshold) {
452 if (!mManager) {
453 return SkipAccessPointPromise::CreateAndReject(
454 SkipFailureHolder(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
455 RESULT_DETAIL("manager is detached.")),
456 0),
457 __func__);
458 }
459
460 MOZ_ASSERT(OnTaskQueue());
461 uint32_t parsed = 0;
462 // Ensure that the data we are about to skip to is still available.
463 TimeIntervals buffered = mManager->Buffered(mType);
464 buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
465 if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) {
466 bool found;
467 parsed = mManager->SkipToNextRandomAccessPoint(
468 mType, aTimeThreadshold, MediaSourceDemuxer::EOS_FUZZ, found);
469 if (found) {
470 return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
471 }
472 }
473 SkipFailureHolder holder(mManager->IsEnded()
474 ? NS_ERROR_DOM_MEDIA_END_OF_STREAM
475 : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
476 parsed);
477 return SkipAccessPointPromise::CreateAndReject(holder, __func__);
478 }
479
HasManager(TrackBuffersManager * aManager) const480 bool MediaSourceTrackDemuxer::HasManager(TrackBuffersManager* aManager) const {
481 MOZ_ASSERT(OnTaskQueue());
482 return mManager == aManager;
483 }
484
DetachManager()485 void MediaSourceTrackDemuxer::DetachManager() {
486 MOZ_ASSERT(OnTaskQueue());
487 MonitorAutoLock mon(mMonitor);
488 mManager = nullptr;
489 }
490
491 } // namespace mozilla
492