1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MediaDecoderReader.h"
8 #include "AbstractMediaDecoder.h"
9 #include "MediaResource.h"
10 #include "VideoUtils.h"
11 #include "ImageContainer.h"
12 #include "MediaPrefs.h"
13
14 #include "nsPrintfCString.h"
15 #include "mozilla/mozalloc.h"
16 #include "mozilla/Mutex.h"
17 #include <stdint.h>
18 #include <algorithm>
19
20 using namespace mozilla::media;
21
22 namespace mozilla {
23
24 // Un-comment to enable logging of seek bisections.
25 //#define SEEK_LOGGING
26
27 extern LazyLogModule gMediaDecoderLog;
28
29 // avoid redefined macro in unified build
30 #undef FMT
31 #undef DECODER_LOG
32 #undef DECODER_WARN
33
34 #define FMT(x, ...) "Decoder=%p " x, mDecoder, ##__VA_ARGS__
35 #define DECODER_LOG(...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(__VA_ARGS__)))
36 #define DECODER_WARN(...) NS_WARNING(nsPrintfCString(FMT(__VA_ARGS__)).get())
37
38 class VideoQueueMemoryFunctor : public nsDequeFunctor {
39 public:
VideoQueueMemoryFunctor()40 VideoQueueMemoryFunctor() : mSize(0) {}
41
42 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
43
operator ()(void * aObject)44 virtual void* operator()(void* aObject) {
45 const VideoData* v = static_cast<const VideoData*>(aObject);
46 mSize += v->SizeOfIncludingThis(MallocSizeOf);
47 return nullptr;
48 }
49
50 size_t mSize;
51 };
52
53
54 class AudioQueueMemoryFunctor : public nsDequeFunctor {
55 public:
AudioQueueMemoryFunctor()56 AudioQueueMemoryFunctor() : mSize(0) {}
57
58 MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf);
59
operator ()(void * aObject)60 virtual void* operator()(void* aObject) {
61 const AudioData* audioData = static_cast<const AudioData*>(aObject);
62 mSize += audioData->SizeOfIncludingThis(MallocSizeOf);
63 return nullptr;
64 }
65
66 size_t mSize;
67 };
68
MediaDecoderReader(AbstractMediaDecoder * aDecoder)69 MediaDecoderReader::MediaDecoderReader(AbstractMediaDecoder* aDecoder)
70 : mAudioCompactor(mAudioQueue)
71 , mDecoder(aDecoder)
72 , mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
73 /* aSupportsTailDispatch = */ true))
74 , mWatchManager(this, mTaskQueue)
75 , mBuffered(mTaskQueue, TimeIntervals(), "MediaDecoderReader::mBuffered (Canonical)")
76 , mDuration(mTaskQueue, NullableTimeUnit(), "MediaDecoderReader::mDuration (Mirror)")
77 , mIgnoreAudioOutputFormat(false)
78 , mHitAudioDecodeError(false)
79 , mShutdown(false)
80 {
81 MOZ_COUNT_CTOR(MediaDecoderReader);
82 MOZ_ASSERT(NS_IsMainThread());
83 }
84
85 nsresult
Init()86 MediaDecoderReader::Init()
87 {
88 if (mDecoder && mDecoder->DataArrivedEvent()) {
89 mDataArrivedListener = mDecoder->DataArrivedEvent()->Connect(
90 mTaskQueue, this, &MediaDecoderReader::NotifyDataArrived);
91 }
92 // Dispatch initialization that needs to happen on that task queue.
93 mTaskQueue->Dispatch(NewRunnableMethod(this, &MediaDecoderReader::InitializationTask));
94 return InitInternal();
95 }
96
97 void
InitializationTask()98 MediaDecoderReader::InitializationTask()
99 {
100 if (!mDecoder) {
101 return;
102 }
103 if (mDecoder->CanonicalDurationOrNull()) {
104 mDuration.Connect(mDecoder->CanonicalDurationOrNull());
105 }
106
107 // Initialize watchers.
108 mWatchManager.Watch(mDuration, &MediaDecoderReader::UpdateBuffered);
109 }
110
~MediaDecoderReader()111 MediaDecoderReader::~MediaDecoderReader()
112 {
113 MOZ_ASSERT(mShutdown);
114 MOZ_COUNT_DTOR(MediaDecoderReader);
115 }
116
SizeOfVideoQueueInBytes() const117 size_t MediaDecoderReader::SizeOfVideoQueueInBytes() const
118 {
119 VideoQueueMemoryFunctor functor;
120 mVideoQueue.LockedForEach(functor);
121 return functor.mSize;
122 }
123
SizeOfAudioQueueInBytes() const124 size_t MediaDecoderReader::SizeOfAudioQueueInBytes() const
125 {
126 AudioQueueMemoryFunctor functor;
127 mAudioQueue.LockedForEach(functor);
128 return functor.mSize;
129 }
130
SizeOfVideoQueueInFrames()131 size_t MediaDecoderReader::SizeOfVideoQueueInFrames()
132 {
133 return mVideoQueue.GetSize();
134 }
135
SizeOfAudioQueueInFrames()136 size_t MediaDecoderReader::SizeOfAudioQueueInFrames()
137 {
138 return mAudioQueue.GetSize();
139 }
140
ResetDecode(TrackSet aTracks)141 nsresult MediaDecoderReader::ResetDecode(TrackSet aTracks)
142 {
143 if (aTracks.contains(TrackInfo::kVideoTrack)) {
144 VideoQueue().Reset();
145 mBaseVideoPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
146 }
147
148 if (aTracks.contains(TrackInfo::kAudioTrack)) {
149 AudioQueue().Reset();
150 mBaseAudioPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
151 }
152
153 return NS_OK;
154 }
155
156 RefPtr<MediaDecoderReader::MediaDataPromise>
DecodeToFirstVideoData()157 MediaDecoderReader::DecodeToFirstVideoData()
158 {
159 MOZ_ASSERT(OnTaskQueue());
160 typedef MediaDecoderReader::MediaDataPromise PromiseType;
161 RefPtr<PromiseType::Private> p = new PromiseType::Private(__func__);
162 RefPtr<MediaDecoderReader> self = this;
163 InvokeUntil([self] () -> bool {
164 MOZ_ASSERT(self->OnTaskQueue());
165 NS_ENSURE_TRUE(!self->mShutdown, false);
166 bool skip = false;
167 if (!self->DecodeVideoFrame(skip, 0)) {
168 self->VideoQueue().Finish();
169 return !!self->VideoQueue().PeekFront();
170 }
171 return true;
172 }, [self] () -> bool {
173 MOZ_ASSERT(self->OnTaskQueue());
174 return self->VideoQueue().GetSize();
175 })->Then(OwnerThread(), __func__, [self, p] () {
176 p->Resolve(self->VideoQueue().PeekFront(), __func__);
177 }, [p] () {
178 // We don't have a way to differentiate EOS, error, and shutdown here. :-(
179 p->Reject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
180 });
181
182 return p.forget();
183 }
184
185 void
UpdateBuffered()186 MediaDecoderReader::UpdateBuffered()
187 {
188 MOZ_ASSERT(OnTaskQueue());
189 NS_ENSURE_TRUE_VOID(!mShutdown);
190 mBuffered = GetBuffered();
191 }
192
193 void
VisibilityChanged()194 MediaDecoderReader::VisibilityChanged()
195 {}
196
197 media::TimeIntervals
GetBuffered()198 MediaDecoderReader::GetBuffered()
199 {
200 MOZ_ASSERT(OnTaskQueue());
201 if (!HaveStartTime()) {
202 return media::TimeIntervals();
203 }
204 AutoPinned<MediaResource> stream(mDecoder->GetResource());
205
206 if (!mDuration.Ref().isSome()) {
207 return TimeIntervals();
208 }
209
210 return GetEstimatedBufferedTimeRanges(stream, mDuration.Ref().ref().ToMicroseconds());
211 }
212
213 RefPtr<MediaDecoderReader::MetadataPromise>
AsyncReadMetadata()214 MediaDecoderReader::AsyncReadMetadata()
215 {
216 MOZ_ASSERT(OnTaskQueue());
217 DECODER_LOG("MediaDecoderReader::AsyncReadMetadata");
218
219 // Attempt to read the metadata.
220 RefPtr<MetadataHolder> metadata = new MetadataHolder();
221 nsresult rv = ReadMetadata(&metadata->mInfo, getter_Transfers(metadata->mTags));
222 metadata->mInfo.AssertValid();
223
224 // We're not waiting for anything. If we didn't get the metadata, that's an
225 // error.
226 if (NS_FAILED(rv) || !metadata->mInfo.HasValidMedia()) {
227 DECODER_WARN("ReadMetadata failed, rv=%x HasValidMedia=%d", rv, metadata->mInfo.HasValidMedia());
228 return MetadataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
229 }
230
231 // Success!
232 return MetadataPromise::CreateAndResolve(metadata, __func__);
233 }
234
235 class ReRequestVideoWithSkipTask : public Runnable
236 {
237 public:
ReRequestVideoWithSkipTask(MediaDecoderReader * aReader,int64_t aTimeThreshold)238 ReRequestVideoWithSkipTask(MediaDecoderReader* aReader,
239 int64_t aTimeThreshold)
240 : mReader(aReader)
241 , mTimeThreshold(aTimeThreshold)
242 {
243 }
244
Run()245 NS_IMETHOD Run() override
246 {
247 MOZ_ASSERT(mReader->OnTaskQueue());
248
249 // Make sure ResetDecode hasn't been called in the mean time.
250 if (!mReader->mBaseVideoPromise.IsEmpty()) {
251 mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold);
252 }
253
254 return NS_OK;
255 }
256
257 private:
258 RefPtr<MediaDecoderReader> mReader;
259 const int64_t mTimeThreshold;
260 };
261
262 class ReRequestAudioTask : public Runnable
263 {
264 public:
ReRequestAudioTask(MediaDecoderReader * aReader)265 explicit ReRequestAudioTask(MediaDecoderReader* aReader)
266 : mReader(aReader)
267 {
268 }
269
Run()270 NS_IMETHOD Run() override
271 {
272 MOZ_ASSERT(mReader->OnTaskQueue());
273
274 // Make sure ResetDecode hasn't been called in the mean time.
275 if (!mReader->mBaseAudioPromise.IsEmpty()) {
276 mReader->RequestAudioData();
277 }
278
279 return NS_OK;
280 }
281
282 private:
283 RefPtr<MediaDecoderReader> mReader;
284 };
285
286 RefPtr<MediaDecoderReader::MediaDataPromise>
RequestVideoData(bool aSkipToNextKeyframe,int64_t aTimeThreshold)287 MediaDecoderReader::RequestVideoData(bool aSkipToNextKeyframe,
288 int64_t aTimeThreshold)
289 {
290 RefPtr<MediaDataPromise> p = mBaseVideoPromise.Ensure(__func__);
291 bool skip = aSkipToNextKeyframe;
292 while (VideoQueue().GetSize() == 0 &&
293 !VideoQueue().IsFinished()) {
294 if (!DecodeVideoFrame(skip, aTimeThreshold)) {
295 VideoQueue().Finish();
296 } else if (skip) {
297 // We still need to decode more data in order to skip to the next
298 // keyframe. Post another task to the decode task queue to decode
299 // again. We don't just decode straight in a loop here, as that
300 // would hog the decode task queue.
301 RefPtr<nsIRunnable> task(new ReRequestVideoWithSkipTask(this, aTimeThreshold));
302 mTaskQueue->Dispatch(task.forget());
303 return p;
304 }
305 }
306 if (VideoQueue().GetSize() > 0) {
307 RefPtr<VideoData> v = VideoQueue().PopFront();
308 mBaseVideoPromise.Resolve(v, __func__);
309 } else if (VideoQueue().IsFinished()) {
310 mBaseVideoPromise.Reject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
311 } else {
312 MOZ_ASSERT(false, "Dropping this promise on the floor");
313 }
314
315 return p;
316 }
317
318 RefPtr<MediaDecoderReader::MediaDataPromise>
RequestAudioData()319 MediaDecoderReader::RequestAudioData()
320 {
321 RefPtr<MediaDataPromise> p = mBaseAudioPromise.Ensure(__func__);
322 while (AudioQueue().GetSize() == 0 &&
323 !AudioQueue().IsFinished()) {
324 if (!DecodeAudioData()) {
325 AudioQueue().Finish();
326 break;
327 }
328 // AudioQueue size is still zero, post a task to try again. Don't spin
329 // waiting in this while loop since it somehow prevents audio EOS from
330 // coming in gstreamer 1.x when there is still video buffer waiting to be
331 // consumed. (|mVideoSinkBufferCount| > 0)
332 if (AudioQueue().GetSize() == 0) {
333 RefPtr<nsIRunnable> task(new ReRequestAudioTask(this));
334 mTaskQueue->Dispatch(task.forget());
335 return p;
336 }
337 }
338 if (AudioQueue().GetSize() > 0) {
339 RefPtr<AudioData> a = AudioQueue().PopFront();
340 mBaseAudioPromise.Resolve(a, __func__);
341 } else if (AudioQueue().IsFinished()) {
342 mBaseAudioPromise.Reject(mHitAudioDecodeError
343 ? NS_ERROR_DOM_MEDIA_FATAL_ERR
344 : NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
345 mHitAudioDecodeError = false;
346 } else {
347 MOZ_ASSERT(false, "Dropping this promise on the floor");
348 }
349
350 return p;
351 }
352
353 RefPtr<ShutdownPromise>
Shutdown()354 MediaDecoderReader::Shutdown()
355 {
356 MOZ_ASSERT(OnTaskQueue());
357 mShutdown = true;
358
359 mBaseAudioPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
360 mBaseVideoPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__);
361
362 mDataArrivedListener.DisconnectIfExists();
363
364 ReleaseResources();
365 mDuration.DisconnectIfConnected();
366 mBuffered.DisconnectAll();
367
368 // Shut down the watch manager before shutting down our task queue.
369 mWatchManager.Shutdown();
370
371 mDecoder = nullptr;
372
373 return mTaskQueue->BeginShutdown();
374 }
375
376 } // namespace mozilla
377