1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set ts=2 sw=2 sts=2 et cindent: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "OmxDataDecoder.h"
8
9 #include "OMX_Audio.h"
10 #include "OMX_Component.h"
11 #include "OMX_Types.h"
12 #include "OmxPlatformLayer.h"
13 #include "mozilla/IntegerPrintfMacros.h"
14
15 #ifdef LOG
16 # undef LOG
17 # undef LOGL
18 #endif
19
20 #define LOG(arg, ...) \
21 DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
22 ##__VA_ARGS__)
23
24 #define LOGL(arg, ...) \
25 DDMOZ_LOGEX(self.get(), sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, \
26 __func__, ##__VA_ARGS__)
27
28 #define CHECK_OMX_ERR(err) \
29 if (err != OMX_ErrorNone) { \
30 NotifyError(err, __func__); \
31 return; \
32 }
33
34 namespace mozilla {
35
36 using namespace gfx;
37
StateTypeToStr(OMX_STATETYPE aType)38 static const char* StateTypeToStr(OMX_STATETYPE aType) {
39 MOZ_ASSERT(aType == OMX_StateLoaded || aType == OMX_StateIdle ||
40 aType == OMX_StateExecuting || aType == OMX_StatePause ||
41 aType == OMX_StateWaitForResources || aType == OMX_StateInvalid);
42
43 switch (aType) {
44 case OMX_StateLoaded:
45 return "OMX_StateLoaded";
46 case OMX_StateIdle:
47 return "OMX_StateIdle";
48 case OMX_StateExecuting:
49 return "OMX_StateExecuting";
50 case OMX_StatePause:
51 return "OMX_StatePause";
52 case OMX_StateWaitForResources:
53 return "OMX_StateWaitForResources";
54 case OMX_StateInvalid:
55 return "OMX_StateInvalid";
56 default:
57 return "Unknown";
58 }
59 }
60
61 // A helper class to retrieve AudioData or VideoData.
62 class MediaDataHelper {
63 protected:
64 virtual ~MediaDataHelper() = default;
65
66 public:
67 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper)
68
69 MediaDataHelper(const TrackInfo* aTrackInfo,
70 layers::ImageContainer* aImageContainer,
71 OmxPromiseLayer* aOmxLayer);
72
73 already_AddRefed<MediaData> GetMediaData(BufferData* aBufferData,
74 bool& aPlatformDepenentData);
75
76 protected:
77 already_AddRefed<AudioData> CreateAudioData(BufferData* aBufferData);
78
79 already_AddRefed<VideoData> CreateYUV420VideoData(BufferData* aBufferData);
80
81 const TrackInfo* mTrackInfo;
82
83 OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef;
84
85 // audio output
86 MediaQueue<AudioData> mAudioQueue;
87
88 AudioCompactor mAudioCompactor;
89
90 // video output
91 RefPtr<layers::ImageContainer> mImageContainer;
92 };
93
OmxDataDecoder(const TrackInfo & aTrackInfo,layers::ImageContainer * aImageContainer)94 OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo,
95 layers::ImageContainer* aImageContainer)
96 : mOmxTaskQueue(
97 CreateMediaDecodeTaskQueue("OmxDataDecoder::mOmxTaskQueue")),
98 mImageContainer(aImageContainer),
99 mWatchManager(this, mOmxTaskQueue),
100 mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState"),
101 mTrackInfo(aTrackInfo.Clone()),
102 mFlushing(false),
103 mShuttingDown(false),
104 mCheckingInputExhausted(false),
105 mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged") {
106 LOG("");
107 mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer);
108 }
109
~OmxDataDecoder()110 OmxDataDecoder::~OmxDataDecoder() { LOG(""); }
111
InitializationTask()112 void OmxDataDecoder::InitializationTask() {
113 mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner);
114 mWatchManager.Watch(mPortSettingsChanged,
115 &OmxDataDecoder::PortSettingsChanged);
116 }
117
EndOfStream()118 void OmxDataDecoder::EndOfStream() {
119 LOG("");
120 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
121
122 RefPtr<OmxDataDecoder> self = this;
123 mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
124 ->Then(mOmxTaskQueue, __func__,
125 [self, this](OmxCommandPromise::ResolveOrRejectValue&& aValue) {
126 mDrainPromise.ResolveIfExists(std::move(mDecodedData), __func__);
127 mDecodedData = DecodedData();
128 });
129 }
130
Init()131 RefPtr<MediaDataDecoder::InitPromise> OmxDataDecoder::Init() {
132 LOG("");
133
134 mThread = GetCurrentSerialEventTarget();
135 RefPtr<OmxDataDecoder> self = this;
136 return InvokeAsync(mOmxTaskQueue, __func__, [self, this]() {
137 InitializationTask();
138
139 RefPtr<InitPromise> p = mInitPromise.Ensure(__func__);
140 mOmxLayer->Init(mTrackInfo.get())
141 ->Then(
142 mOmxTaskQueue, __func__,
143 [self, this]() {
144 // Omx state should be OMX_StateIdle.
145 mOmxState = mOmxLayer->GetState();
146 MOZ_ASSERT(mOmxState != OMX_StateIdle);
147 },
148 [self, this]() {
149 RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
150 });
151 return p;
152 });
153 }
154
Decode(MediaRawData * aSample)155 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Decode(
156 MediaRawData* aSample) {
157 LOG("sample %p", aSample);
158 MOZ_ASSERT(mThread->IsOnCurrentThread());
159 MOZ_ASSERT(mInitPromise.IsEmpty());
160
161 RefPtr<OmxDataDecoder> self = this;
162 RefPtr<MediaRawData> sample = aSample;
163 return InvokeAsync(mOmxTaskQueue, __func__, [self, this, sample]() {
164 RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
165 mMediaRawDatas.AppendElement(std::move(sample));
166
167 // Start to fill/empty buffers.
168 if (mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting) {
169 FillAndEmptyBuffers();
170 }
171 return p;
172 });
173 }
174
Flush()175 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::Flush() {
176 LOG("");
177 MOZ_ASSERT(mThread->IsOnCurrentThread());
178
179 mFlushing = true;
180
181 return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoFlush);
182 }
183
Drain()184 RefPtr<MediaDataDecoder::DecodePromise> OmxDataDecoder::Drain() {
185 LOG("");
186 MOZ_ASSERT(mThread->IsOnCurrentThread());
187
188 RefPtr<OmxDataDecoder> self = this;
189 return InvokeAsync(mOmxTaskQueue, __func__, [self]() {
190 RefPtr<DecodePromise> p = self->mDrainPromise.Ensure(__func__);
191 self->SendEosBuffer();
192 return p;
193 });
194 }
195
Shutdown()196 RefPtr<ShutdownPromise> OmxDataDecoder::Shutdown() {
197 LOG("");
198 // mThread may not be set if Init hasn't been called first.
199 MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread());
200
201 mShuttingDown = true;
202
203 return InvokeAsync(mOmxTaskQueue, this, __func__,
204 &OmxDataDecoder::DoAsyncShutdown);
205 }
206
DoAsyncShutdown()207 RefPtr<ShutdownPromise> OmxDataDecoder::DoAsyncShutdown() {
208 LOG("");
209 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
210 MOZ_ASSERT(!mFlushing);
211
212 mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner);
213 mWatchManager.Unwatch(mPortSettingsChanged,
214 &OmxDataDecoder::PortSettingsChanged);
215
216 // Flush to all ports, so all buffers can be returned from component.
217 RefPtr<OmxDataDecoder> self = this;
218 mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
219 ->Then(
220 mOmxTaskQueue, __func__,
221 [self]() -> RefPtr<OmxCommandPromise> {
222 LOGL("DoAsyncShutdown: flush complete");
223 return self->mOmxLayer->SendCommand(OMX_CommandStateSet,
224 OMX_StateIdle, nullptr);
225 },
226 [self](const OmxCommandFailureHolder& aError) {
227 self->mOmxLayer->Shutdown();
228 return OmxCommandPromise::CreateAndReject(aError, __func__);
229 })
230 ->Then(
231 mOmxTaskQueue, __func__,
232 [self]() -> RefPtr<OmxCommandPromise> {
233 RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
234 OMX_CommandStateSet, OMX_StateLoaded, nullptr);
235
236 // According to spec 3.1.1.2.2.1:
237 // OMX_StateLoaded needs to be sent before releasing buffers.
238 // And state transition from OMX_StateIdle to OMX_StateLoaded
239 // is completed when all of the buffers have been removed
240 // from the component.
241 // Here the buffer promises are not resolved due to displaying
242 // in layer, it needs to wait before the layer returns the
243 // buffers.
244 LOGL("DoAsyncShutdown: releasing buffers...");
245 self->ReleaseBuffers(OMX_DirInput);
246 self->ReleaseBuffers(OMX_DirOutput);
247
248 return p;
249 },
250 [self](const OmxCommandFailureHolder& aError) {
251 self->mOmxLayer->Shutdown();
252 return OmxCommandPromise::CreateAndReject(aError, __func__);
253 })
254 ->Then(
255 mOmxTaskQueue, __func__,
256 [self]() -> RefPtr<ShutdownPromise> {
257 LOGL(
258 "DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx");
259 self->mOmxLayer->Shutdown();
260 self->mWatchManager.Shutdown();
261 self->mOmxLayer = nullptr;
262 self->mMediaDataHelper = nullptr;
263 self->mShuttingDown = false;
264 return ShutdownPromise::CreateAndResolve(true, __func__);
265 },
266 [self]() -> RefPtr<ShutdownPromise> {
267 self->mOmxLayer->Shutdown();
268 self->mWatchManager.Shutdown();
269 self->mOmxLayer = nullptr;
270 self->mMediaDataHelper = nullptr;
271 return ShutdownPromise::CreateAndReject(false, __func__);
272 })
273 ->Then(
274 mThread, __func__,
275 [self]() {
276 self->mOmxTaskQueue->BeginShutdown();
277 self->mOmxTaskQueue->AwaitShutdownAndIdle();
278 self->mShutdownPromise.Resolve(true, __func__);
279 },
280 [self]() {
281 self->mOmxTaskQueue->BeginShutdown();
282 self->mOmxTaskQueue->AwaitShutdownAndIdle();
283 self->mShutdownPromise.Resolve(true, __func__);
284 });
285 return mShutdownPromise.Ensure(__func__);
286 }
287
FillBufferDone(BufferData * aData)288 void OmxDataDecoder::FillBufferDone(BufferData* aData) {
289 MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
290
291 // Don't output sample when flush or shutting down, especially for video
292 // decoded frame. Because video decoded frame can have a promise in
293 // BufferData waiting for layer to resolve it via recycle callback, if other
294 // module doesn't send it to layer, it will cause a unresolved promise and
295 // waiting for resolve infinitely.
296 if (mFlushing || mShuttingDown) {
297 LOG("mFlush or mShuttingDown, drop data");
298 aData->mStatus = BufferData::BufferStatus::FREE;
299 return;
300 }
301
302 if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) {
303 // Reach eos, it's an empty data so it doesn't need to output.
304 EndOfStream();
305 aData->mStatus = BufferData::BufferStatus::FREE;
306 } else {
307 Output(aData);
308 FillAndEmptyBuffers();
309 }
310 }
311
Output(BufferData * aData)312 void OmxDataDecoder::Output(BufferData* aData) {
313 if (!mMediaDataHelper) {
314 mMediaDataHelper =
315 new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer);
316 }
317
318 bool isPlatformData = false;
319 RefPtr<MediaData> data =
320 mMediaDataHelper->GetMediaData(aData, isPlatformData);
321 if (!data) {
322 aData->mStatus = BufferData::BufferStatus::FREE;
323 return;
324 }
325
326 if (isPlatformData) {
327 // If the MediaData is platform dependnet data, it's mostly a kind of
328 // limited resource, so we use promise to notify when the resource is free.
329 aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT;
330
331 MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty());
332 RefPtr<OmxBufferPromise> p = aData->mPromise.Ensure(__func__);
333
334 RefPtr<OmxDataDecoder> self = this;
335 RefPtr<BufferData> buffer = aData;
336 p->Then(
337 mOmxTaskQueue, __func__,
338 [self, buffer]() {
339 MOZ_RELEASE_ASSERT(buffer->mStatus ==
340 BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
341 buffer->mStatus = BufferData::BufferStatus::FREE;
342 self->FillAndEmptyBuffers();
343 },
344 [buffer]() {
345 MOZ_RELEASE_ASSERT(buffer->mStatus ==
346 BufferData::BufferStatus::OMX_CLIENT_OUTPUT);
347 buffer->mStatus = BufferData::BufferStatus::FREE;
348 });
349 } else {
350 aData->mStatus = BufferData::BufferStatus::FREE;
351 }
352
353 mDecodedData.AppendElement(std::move(data));
354 }
355
FillBufferFailure(OmxBufferFailureHolder aFailureHolder)356 void OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) {
357 NotifyError(aFailureHolder.mError, __func__);
358 }
359
EmptyBufferDone(BufferData * aData)360 void OmxDataDecoder::EmptyBufferDone(BufferData* aData) {
361 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
362 MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT);
363
364 // Nothing to do when status of input buffer is OMX_CLIENT.
365 aData->mStatus = BufferData::BufferStatus::FREE;
366 FillAndEmptyBuffers();
367
368 // There is no way to know if component gets enough raw samples to generate
369 // output, especially for video decoding. So here it needs to request raw
370 // samples aggressively.
371 if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) {
372 mCheckingInputExhausted = true;
373
374 RefPtr<OmxDataDecoder> self = this;
375 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
376 "OmxDataDecoder::EmptyBufferDone", [self, this]() {
377 mCheckingInputExhausted = false;
378
379 if (mMediaRawDatas.Length()) {
380 return;
381 }
382
383 mDecodePromise.ResolveIfExists(std::move(mDecodedData), __func__);
384 mDecodedData = DecodedData();
385 });
386
387 nsresult rv = mOmxTaskQueue->Dispatch(r.forget());
388 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
389 Unused << rv;
390 }
391 }
392
EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder)393 void OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) {
394 NotifyError(aFailureHolder.mError, __func__);
395 }
396
NotifyError(OMX_ERRORTYPE aOmxError,const char * aLine,const MediaResult & aError)397 void OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine,
398 const MediaResult& aError) {
399 LOG("NotifyError %d (%s) at %s", static_cast<int>(aOmxError),
400 aError.ErrorName().get(), aLine);
401 mDecodedData = DecodedData();
402 mDecodePromise.RejectIfExists(aError, __func__);
403 mDrainPromise.RejectIfExists(aError, __func__);
404 mFlushPromise.RejectIfExists(aError, __func__);
405 }
406
FillAndEmptyBuffers()407 void OmxDataDecoder::FillAndEmptyBuffers() {
408 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
409 MOZ_ASSERT(mOmxState == OMX_StateExecuting);
410
411 // During the port setting changed, it is forbidden to do any buffer
412 // operation.
413 if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) {
414 return;
415 }
416
417 // Trigger input port.
418 while (!!mMediaRawDatas.Length()) {
419 // input buffer must be used by component if there is data available.
420 RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
421 if (!inbuf) {
422 LOG("no input buffer!");
423 break;
424 }
425
426 RefPtr<MediaRawData> data = mMediaRawDatas[0];
427 // Buffer size should large enough for raw data.
428 MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size());
429
430 memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size());
431 inbuf->mBuffer->nFilledLen = data->Size();
432 inbuf->mBuffer->nOffset = 0;
433 inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size()
434 ? OMX_BUFFERFLAG_ENDOFFRAME
435 : 0;
436 inbuf->mBuffer->nTimeStamp = data->mTime.ToMicroseconds();
437 if (data->Size()) {
438 inbuf->mRawData = mMediaRawDatas[0];
439 } else {
440 LOG("send EOS buffer");
441 inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS;
442 }
443
444 LOG("feed sample %p to omx component, len %ld, flag %lX", data.get(),
445 inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags);
446 mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
447 &OmxDataDecoder::EmptyBufferDone,
448 &OmxDataDecoder::EmptyBufferFailure);
449 mMediaRawDatas.RemoveElementAt(0);
450 }
451
452 // Trigger output port.
453 while (true) {
454 RefPtr<BufferData> outbuf = FindAvailableBuffer(OMX_DirOutput);
455 if (!outbuf) {
456 break;
457 }
458
459 mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this,
460 &OmxDataDecoder::FillBufferDone,
461 &OmxDataDecoder::FillBufferFailure);
462 }
463 }
464
FindAvailableBuffer(OMX_DIRTYPE aType)465 OmxPromiseLayer::BufferData* OmxDataDecoder::FindAvailableBuffer(
466 OMX_DIRTYPE aType) {
467 BUFFERLIST* buffers = GetBuffers(aType);
468
469 for (uint32_t i = 0; i < buffers->Length(); i++) {
470 BufferData* buf = buffers->ElementAt(i);
471 if (buf->mStatus == BufferData::BufferStatus::FREE) {
472 return buf;
473 }
474 }
475
476 return nullptr;
477 }
478
AllocateBuffers(OMX_DIRTYPE aType)479 nsresult OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) {
480 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
481
482 return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType));
483 }
484
ReleaseBuffers(OMX_DIRTYPE aType)485 nsresult OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) {
486 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
487
488 return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType));
489 }
490
GetBuffers(OMX_DIRTYPE aType)491 nsTArray<RefPtr<OmxPromiseLayer::BufferData>>* OmxDataDecoder::GetBuffers(
492 OMX_DIRTYPE aType) {
493 MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput ||
494 aType == OMX_DIRTYPE::OMX_DirOutput);
495
496 if (aType == OMX_DIRTYPE::OMX_DirInput) {
497 return &mInPortBuffers;
498 }
499 return &mOutPortBuffers;
500 }
501
ResolveInitPromise(const char * aMethodName)502 void OmxDataDecoder::ResolveInitPromise(const char* aMethodName) {
503 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
504 LOG("called from %s", aMethodName);
505 mInitPromise.ResolveIfExists(mTrackInfo->GetType(), aMethodName);
506 }
507
RejectInitPromise(MediaResult aError,const char * aMethodName)508 void OmxDataDecoder::RejectInitPromise(MediaResult aError,
509 const char* aMethodName) {
510 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
511 mInitPromise.RejectIfExists(aError, aMethodName);
512 }
513
OmxStateRunner()514 void OmxDataDecoder::OmxStateRunner() {
515 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
516 LOG("OMX state: %s", StateTypeToStr(mOmxState));
517
518 // TODO: maybe it'd be better to use promise CompletionPromise() to replace
519 // this state machine.
520 if (mOmxState == OMX_StateLoaded) {
521 ConfigCodec();
522
523 // Send OpenMax state command to OMX_StateIdle.
524 RefPtr<OmxDataDecoder> self = this;
525 mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr)
526 ->Then(
527 mOmxTaskQueue, __func__,
528 [self]() {
529 // Current state should be OMX_StateIdle.
530 self->mOmxState = self->mOmxLayer->GetState();
531 MOZ_ASSERT(self->mOmxState == OMX_StateIdle);
532 },
533 [self]() {
534 self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
535 });
536
537 // Allocate input and output buffers.
538 OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput,
539 OMX_DIRTYPE::OMX_DirOutput};
540 for (const auto id : types) {
541 if (NS_FAILED(AllocateBuffers(id))) {
542 LOG("Failed to allocate buffer on port %d", id);
543 RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
544 break;
545 }
546 }
547 } else if (mOmxState == OMX_StateIdle) {
548 RefPtr<OmxDataDecoder> self = this;
549 mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr)
550 ->Then(
551 mOmxTaskQueue, __func__,
552 [self]() {
553 self->mOmxState = self->mOmxLayer->GetState();
554 MOZ_ASSERT(self->mOmxState == OMX_StateExecuting);
555
556 self->ResolveInitPromise(__func__);
557 },
558 [self]() {
559 self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
560 });
561 } else if (mOmxState == OMX_StateExecuting) {
562 // Configure codec once it gets OMX_StateExecuting state.
563 FillCodecConfigDataToOmx();
564 } else {
565 MOZ_ASSERT(0);
566 }
567 }
568
ConfigCodec()569 void OmxDataDecoder::ConfigCodec() {
570 OMX_ERRORTYPE err = mOmxLayer->Config();
571 CHECK_OMX_ERR(err);
572 }
573
FillCodecConfigDataToOmx()574 void OmxDataDecoder::FillCodecConfigDataToOmx() {
575 // Codec configure data should be the first sample running on Omx TaskQueue.
576 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
577 MOZ_ASSERT(!mMediaRawDatas.Length());
578 MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting);
579
580 RefPtr<BufferData> inbuf = FindAvailableBuffer(OMX_DirInput);
581 RefPtr<MediaByteBuffer> csc;
582 if (mTrackInfo->IsAudio()) {
583 csc = mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig;
584 } else if (mTrackInfo->IsVideo()) {
585 csc = mTrackInfo->GetAsVideoInfo()->mExtraData;
586 }
587
588 MOZ_RELEASE_ASSERT(csc);
589
590 // Some codecs like h264, its codec specific data is at the first packet, not
591 // in container.
592 if (csc->Length()) {
593 // Buffer size should large enough for raw data.
594 MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= csc->Length());
595
596 memcpy(inbuf->mBuffer->pBuffer, csc->Elements(), csc->Length());
597 inbuf->mBuffer->nFilledLen = csc->Length();
598 inbuf->mBuffer->nOffset = 0;
599 inbuf->mBuffer->nFlags =
600 (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG);
601
602 LOG("Feed codec configure data to OMX component");
603 mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this,
604 &OmxDataDecoder::EmptyBufferDone,
605 &OmxDataDecoder::EmptyBufferFailure);
606 }
607 }
608
Event(OMX_EVENTTYPE aEvent,OMX_U32 aData1,OMX_U32 aData2)609 bool OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1,
610 OMX_U32 aData2) {
611 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
612
613 if (mOmxLayer->Event(aEvent, aData1, aData2)) {
614 return true;
615 }
616
617 switch (aEvent) {
618 case OMX_EventPortSettingsChanged: {
619 // Don't always disable port. See bug 1235340.
620 if (aData2 == 0 || aData2 == OMX_IndexParamPortDefinition) {
621 // According to spec: "To prevent the loss of any input data, the
622 // component issuing the OMX_EventPortSettingsChanged event on its input
623 // port should buffer all input port data that arrives between the
624 // emission of the OMX_EventPortSettingsChanged event and the arrival of
625 // the command to disable the input port."
626 //
627 // So client needs to disable port and reallocate buffers.
628 MOZ_ASSERT(mPortSettingsChanged == -1);
629 mPortSettingsChanged = aData1;
630 }
631 LOG("Got OMX_EventPortSettingsChanged event");
632 break;
633 }
634 default: {
635 // Got error during decoding, send msg to MFR skipping to next key frame.
636 if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) {
637 NotifyError((OMX_ERRORTYPE)aData1, __func__,
638 MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__));
639 return true;
640 }
641 LOG("WARNING: got none handle event: %d, aData1: %ld, aData2: %ld",
642 aEvent, aData1, aData2);
643 return false;
644 }
645 }
646
647 return true;
648 }
649
BuffersCanBeReleased(OMX_DIRTYPE aType)650 bool OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) {
651 BUFFERLIST* buffers = GetBuffers(aType);
652 uint32_t len = buffers->Length();
653 for (uint32_t i = 0; i < len; i++) {
654 BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus;
655 if (buf_status == BufferData::BufferStatus::OMX_COMPONENT ||
656 buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) {
657 return false;
658 }
659 }
660 return true;
661 }
662
663 OMX_DIRTYPE
GetPortDirection(uint32_t aPortIndex)664 OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) {
665 OMX_PARAM_PORTDEFINITIONTYPE def;
666 InitOmxParameter(&def);
667 def.nPortIndex = mPortSettingsChanged;
668
669 OMX_ERRORTYPE err =
670 mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
671 if (err != OMX_ErrorNone) {
672 return OMX_DirMax;
673 }
674 return def.eDir;
675 }
676
677 RefPtr<OmxPromiseLayer::OmxBufferPromise::AllPromiseType>
CollectBufferPromises(OMX_DIRTYPE aType)678 OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) {
679 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
680
681 nsTArray<RefPtr<OmxBufferPromise>> promises;
682 OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput};
683 for (const auto type : types) {
684 if ((aType == type) || (aType == OMX_DirMax)) {
685 // find the buffer which has promise.
686 BUFFERLIST* buffers = GetBuffers(type);
687
688 for (uint32_t i = 0; i < buffers->Length(); i++) {
689 BufferData* buf = buffers->ElementAt(i);
690 if (!buf->mPromise.IsEmpty()) {
691 // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so
692 // it is safe to call "Ensure" here.
693 promises.AppendElement(buf->mPromise.Ensure(__func__));
694 }
695 }
696 }
697 }
698
699 LOG("CollectBufferPromises: type %d, total %zu promiese", aType,
700 promises.Length());
701 if (promises.Length()) {
702 return OmxBufferPromise::All(mOmxTaskQueue, promises);
703 }
704
705 return OmxBufferPromise::AllPromiseType::CreateAndResolve(
706 nsTArray<BufferData*>(), __func__);
707 }
708
PortSettingsChanged()709 void OmxDataDecoder::PortSettingsChanged() {
710 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
711
712 if (mPortSettingsChanged == -1 ||
713 mOmxState == OMX_STATETYPE::OMX_StateInvalid) {
714 return;
715 }
716
717 // The PortSettingsChanged algorithm:
718 //
719 // 1. disable port.
720 // 2. wait for port buffers return to client and then release these buffers.
721 // 3. enable port.
722 // 4. allocate port buffers.
723 //
724
725 // Disable port. Get port definition if the target port is enable.
726 OMX_PARAM_PORTDEFINITIONTYPE def;
727 InitOmxParameter(&def);
728 def.nPortIndex = mPortSettingsChanged;
729
730 OMX_ERRORTYPE err =
731 mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def));
732 CHECK_OMX_ERR(err);
733
734 RefPtr<OmxDataDecoder> self = this;
735 if (def.bEnabled) {
736 // 1. disable port.
737 LOG("PortSettingsChanged: disable port %lu", def.nPortIndex);
738 mOmxLayer
739 ->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr)
740 ->Then(
741 mOmxTaskQueue, __func__,
742 [self, def]() -> RefPtr<OmxCommandPromise> {
743 // 3. enable port.
744 // Send enable port command.
745 RefPtr<OmxCommandPromise> p = self->mOmxLayer->SendCommand(
746 OMX_CommandPortEnable, self->mPortSettingsChanged, nullptr);
747
748 // 4. allocate port buffers.
749 // Allocate new port buffers.
750 nsresult rv = self->AllocateBuffers(def.eDir);
751 if (NS_FAILED(rv)) {
752 self->NotifyError(OMX_ErrorUndefined, __func__);
753 }
754
755 return p;
756 },
757 [self](const OmxCommandFailureHolder& aError) {
758 self->NotifyError(OMX_ErrorUndefined, __func__);
759 return OmxCommandPromise::CreateAndReject(aError, __func__);
760 })
761 ->Then(
762 mOmxTaskQueue, __func__,
763 [self]() {
764 LOGL("PortSettingsChanged: port settings changed complete");
765 // finish port setting changed.
766 self->mPortSettingsChanged = -1;
767 self->FillAndEmptyBuffers();
768 },
769 [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
770
771 // 2. wait for port buffers return to client and then release these buffers.
772 //
773 // Port buffers will be returned to client soon once OMX_CommandPortDisable
774 // command is sent. Then releasing these buffers.
775 CollectBufferPromises(def.eDir)->Then(
776 mOmxTaskQueue, __func__,
777 [self, def]() {
778 MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir));
779 nsresult rv = self->ReleaseBuffers(def.eDir);
780 if (NS_FAILED(rv)) {
781 MOZ_RELEASE_ASSERT(0);
782 self->NotifyError(OMX_ErrorUndefined, __func__);
783 }
784 },
785 [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); });
786 }
787 }
788
SendEosBuffer()789 void OmxDataDecoder::SendEosBuffer() {
790 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
791
792 // There is no 'Drain' API in OpenMax, so it needs to wait for output sample
793 // with EOS flag. However, MediaRawData doesn't provide EOS information,
794 // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in
795 // queue. This behaviour should be compliant with spec, I think...
796 RefPtr<MediaRawData> eos_data = new MediaRawData();
797 mMediaRawDatas.AppendElement(eos_data);
798 FillAndEmptyBuffers();
799 }
800
DoFlush()801 RefPtr<MediaDataDecoder::FlushPromise> OmxDataDecoder::DoFlush() {
802 MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn());
803
804 mDecodedData = DecodedData();
805 mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
806 mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
807
808 RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
809
810 // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue.
811 // 2. Remove all elements in mMediaRawDatas when flush is completed.
812 mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr)
813 ->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FlushComplete,
814 &OmxDataDecoder::FlushFailure);
815
816 return p;
817 }
818
FlushComplete(OMX_COMMANDTYPE aCommandType)819 void OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) {
820 mMediaRawDatas.Clear();
821 mFlushing = false;
822
823 LOG("Flush complete");
824 mFlushPromise.ResolveIfExists(true, __func__);
825 }
826
FlushFailure(OmxCommandFailureHolder aFailureHolder)827 void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) {
828 mFlushing = false;
829 mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
830 }
831
MediaDataHelper(const TrackInfo * aTrackInfo,layers::ImageContainer * aImageContainer,OmxPromiseLayer * aOmxLayer)832 MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo,
833 layers::ImageContainer* aImageContainer,
834 OmxPromiseLayer* aOmxLayer)
835 : mTrackInfo(aTrackInfo),
836 mAudioCompactor(mAudioQueue),
837 mImageContainer(aImageContainer) {
838 InitOmxParameter(&mOutputPortDef);
839 mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex();
840 aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef,
841 sizeof(mOutputPortDef));
842 }
843
GetMediaData(BufferData * aBufferData,bool & aPlatformDepenentData)844 already_AddRefed<MediaData> MediaDataHelper::GetMediaData(
845 BufferData* aBufferData, bool& aPlatformDepenentData) {
846 aPlatformDepenentData = false;
847 RefPtr<MediaData> data;
848
849 if (mTrackInfo->IsAudio()) {
850 if (!aBufferData->mBuffer->nFilledLen) {
851 return nullptr;
852 }
853 data = CreateAudioData(aBufferData);
854 } else if (mTrackInfo->IsVideo()) {
855 data = aBufferData->GetPlatformMediaData();
856 if (data) {
857 aPlatformDepenentData = true;
858 } else {
859 if (!aBufferData->mBuffer->nFilledLen) {
860 return nullptr;
861 }
862 // Get YUV VideoData, it uses more CPU, in most cases, on software codec.
863 data = CreateYUV420VideoData(aBufferData);
864 }
865
866 // Update video time code, duration... from the raw data.
867 VideoData* video(data->As<VideoData>());
868 if (aBufferData->mRawData) {
869 video->mTime = aBufferData->mRawData->mTime;
870 video->mTimecode = aBufferData->mRawData->mTimecode;
871 video->mOffset = aBufferData->mRawData->mOffset;
872 video->mDuration = aBufferData->mRawData->mDuration;
873 video->mKeyframe = aBufferData->mRawData->mKeyframe;
874 }
875 }
876
877 return data.forget();
878 }
879
CreateAudioData(BufferData * aBufferData)880 already_AddRefed<AudioData> MediaDataHelper::CreateAudioData(
881 BufferData* aBufferData) {
882 RefPtr<AudioData> audio;
883 OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer;
884 const AudioInfo* info = mTrackInfo->GetAsAudioInfo();
885 if (buf->nFilledLen) {
886 uint64_t offset = 0;
887 uint32_t frames = buf->nFilledLen / (2 * info->mChannels);
888 if (aBufferData->mRawData) {
889 offset = aBufferData->mRawData->mOffset;
890 }
891 typedef AudioCompactor::NativeCopy OmxCopy;
892 mAudioCompactor.Push(
893 offset, buf->nTimeStamp, info->mRate, frames, info->mChannels,
894 OmxCopy(buf->pBuffer + buf->nOffset, buf->nFilledLen, info->mChannels));
895 audio = mAudioQueue.PopFront();
896 }
897
898 return audio.forget();
899 }
900
CreateYUV420VideoData(BufferData * aBufferData)901 already_AddRefed<VideoData> MediaDataHelper::CreateYUV420VideoData(
902 BufferData* aBufferData) {
903 uint8_t* yuv420p_buffer = (uint8_t*)aBufferData->mBuffer->pBuffer;
904 int32_t stride = mOutputPortDef.format.video.nStride;
905 int32_t slice_height = mOutputPortDef.format.video.nSliceHeight;
906 int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width;
907 int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height;
908
909 // TODO: convert other formats to YUV420.
910 if (mOutputPortDef.format.video.eColorFormat !=
911 OMX_COLOR_FormatYUV420Planar) {
912 return nullptr;
913 }
914
915 size_t yuv420p_y_size = stride * slice_height;
916 size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2);
917 uint8_t* yuv420p_y = yuv420p_buffer;
918 uint8_t* yuv420p_u = yuv420p_y + yuv420p_y_size;
919 uint8_t* yuv420p_v = yuv420p_u + yuv420p_u_size;
920
921 VideoData::YCbCrBuffer b;
922 b.mPlanes[0].mData = yuv420p_y;
923 b.mPlanes[0].mWidth = width;
924 b.mPlanes[0].mHeight = height;
925 b.mPlanes[0].mStride = stride;
926 b.mPlanes[0].mSkip = 0;
927
928 b.mPlanes[1].mData = yuv420p_u;
929 b.mPlanes[1].mWidth = (width + 1) / 2;
930 b.mPlanes[1].mHeight = (height + 1) / 2;
931 b.mPlanes[1].mStride = (stride + 1) / 2;
932 b.mPlanes[1].mSkip = 0;
933
934 b.mPlanes[2].mData = yuv420p_v;
935 b.mPlanes[2].mWidth = (width + 1) / 2;
936 b.mPlanes[2].mHeight = (height + 1) / 2;
937 b.mPlanes[2].mStride = (stride + 1) / 2;
938 b.mPlanes[2].mSkip = 0;
939
940 VideoInfo info(*mTrackInfo->GetAsVideoInfo());
941
942 auto maybeColorSpace = info.mColorSpace;
943 if (!maybeColorSpace) {
944 maybeColorSpace = Some(DefaultColorSpace({width, height}));
945 }
946 b.mYUVColorSpace = *maybeColorSpace;
947
948 RefPtr<VideoData> data = VideoData::CreateAndCopyData(
949 info, mImageContainer,
950 0, // Filled later by caller.
951 media::TimeUnit::Zero(), // Filled later by caller.
952 media::TimeUnit::FromMicroseconds(1), // We don't know the duration.
953 b,
954 0, // Filled later by caller.
955 media::TimeUnit::FromMicroseconds(-1), info.ImageRect());
956
957 MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug,
958 ("YUV420 VideoData: disp width %d, height %d, pic width %d, height "
959 "%d, time %lld",
960 info.mDisplay.width, info.mDisplay.height, info.mImage.width,
961 info.mImage.height, aBufferData->mBuffer->nTimeStamp));
962
963 return data.forget();
964 }
965
966 } // namespace mozilla
967