1 /* This Source Code Form is subject to the terms of the Mozilla Public
2  * License, v. 2.0. If a copy of the MPL was not distributed with this
3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4 
5 #include "AndroidDataEncoder.h"
6 
7 #include "AnnexB.h"
8 #include "MediaData.h"
9 #include "MediaInfo.h"
10 #include "SimpleMap.h"
11 
12 #include "ImageContainer.h"
13 #include "mozilla/Logging.h"
14 #include "mozilla/ResultVariant.h"
15 
16 #include "nsMimeTypes.h"
17 
18 #include "libyuv.h"
19 
20 namespace mozilla {
21 using media::TimeUnit;
22 
23 extern LazyLogModule sPEMLog;
24 #define AND_ENC_LOG(arg, ...)                \
25   MOZ_LOG(sPEMLog, mozilla::LogLevel::Debug, \
26           ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
27 #define AND_ENC_LOGE(arg, ...)               \
28   MOZ_LOG(sPEMLog, mozilla::LogLevel::Error, \
29           ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
30 
31 #define REJECT_IF_ERROR()                                                \
32   do {                                                                   \
33     if (mError) {                                                        \
34       auto error = mError.value();                                       \
35       mError.reset();                                                    \
36       return EncodePromise::CreateAndReject(std::move(error), __func__); \
37     }                                                                    \
38   } while (0)
39 
40 template <typename ConfigType>
Init()41 RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder<ConfigType>::Init() {
42   // Sanity-check the input size for Android software encoder fails to do it.
43   if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
44     return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__);
45   }
46 
47   return InvokeAsync(mTaskQueue, this, __func__,
48                      &AndroidDataEncoder<ConfigType>::ProcessInit);
49 }
50 
MimeTypeOf(MediaDataEncoder::CodecType aCodec)51 static const char* MimeTypeOf(MediaDataEncoder::CodecType aCodec) {
52   switch (aCodec) {
53     case MediaDataEncoder::CodecType::H264:
54       return "video/avc";
55     case MediaDataEncoder::CodecType::VP8:
56       return "video/x-vnd.on2.vp8";
57     case MediaDataEncoder::CodecType::VP9:
58       return "video/x-vnd.on2.vp9";
59     default:
60       return "";
61   }
62 }
63 
64 using FormatResult = Result<java::sdk::MediaFormat::LocalRef, MediaResult>;
65 
66 template <typename ConfigType>
ToMediaFormat(const ConfigType & aConfig)67 FormatResult ToMediaFormat(const ConfigType& aConfig) {
68   nsresult rv = NS_OK;
69   java::sdk::MediaFormat::LocalRef format;
70   rv = java::sdk::MediaFormat::CreateVideoFormat(MimeTypeOf(aConfig.mCodecType),
71                                                  aConfig.mSize.width,
72                                                  aConfig.mSize.height, &format);
73   NS_ENSURE_SUCCESS(
74       rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
75                                    "fail to create Java MediaFormat object")));
76 
77   rv =
78       format->SetInteger(java::sdk::MediaFormat::KEY_BITRATE_MODE, 2 /* CBR */);
79   NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
80                                                  "fail to set bitrate mode")));
81 
82   rv = format->SetInteger(java::sdk::MediaFormat::KEY_BIT_RATE,
83                           aConfig.mBitsPerSec);
84   NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
85                                                  "fail to set bitrate")));
86 
87   // COLOR_FormatYUV420SemiPlanar(NV12) is the most widely supported
88   // format.
89   rv = format->SetInteger(java::sdk::MediaFormat::KEY_COLOR_FORMAT, 0x15);
90   NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
91                                                  "fail to set color format")));
92 
93   rv = format->SetInteger(java::sdk::MediaFormat::KEY_FRAME_RATE,
94                           aConfig.mFramerate);
95   NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
96                                                  "fail to set frame rate")));
97 
98   // Ensure interval >= 1. A negative value means no key frames are
99   // requested after the first frame. A zero value means a stream
100   // containing all key frames is requested.
101   int32_t intervalInSec =
102       std::max<size_t>(1, aConfig.mKeyframeInterval / aConfig.mFramerate);
103   rv = format->SetInteger(java::sdk::MediaFormat::KEY_I_FRAME_INTERVAL,
104                           intervalInSec);
105   NS_ENSURE_SUCCESS(rv,
106                     FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
107                                              "fail to set I-frame interval")));
108 
109   return format;
110 }
111 
112 template <typename ConfigType>
113 RefPtr<MediaDataEncoder::InitPromise>
ProcessInit()114 AndroidDataEncoder<ConfigType>::ProcessInit() {
115   AssertOnTaskQueue();
116   MOZ_ASSERT(!mJavaEncoder);
117 
118   java::sdk::BufferInfo::LocalRef bufferInfo;
119   if (NS_FAILED(java::sdk::BufferInfo::New(&bufferInfo)) || !bufferInfo) {
120     return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
121   }
122   mInputBufferInfo = bufferInfo;
123 
124   FormatResult result = ToMediaFormat<ConfigType>(mConfig);
125   if (result.isErr()) {
126     return InitPromise::CreateAndReject(result.unwrapErr(), __func__);
127   }
128   mFormat = result.unwrap();
129 
130   // Register native methods.
131   JavaCallbacksSupport::Init();
132 
133   mJavaCallbacks = java::CodecProxy::NativeCallbacks::New();
134   if (!mJavaCallbacks) {
135     return InitPromise::CreateAndReject(
136         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
137                     "cannot create Java callback object"),
138         __func__);
139   }
140   JavaCallbacksSupport::AttachNative(
141       mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
142 
143   mJavaEncoder = java::CodecProxy::Create(true /* encoder */, mFormat, nullptr,
144                                           mJavaCallbacks, u""_ns);
145   if (!mJavaEncoder) {
146     return InitPromise::CreateAndReject(
147         MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
148                     "cannot create Java encoder object"),
149         __func__);
150   }
151 
152   mIsHardwareAccelerated = mJavaEncoder->IsHardwareAccelerated();
153   mDrainState = DrainState::DRAINABLE;
154 
155   return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
156 }
157 
158 template <typename ConfigType>
Encode(const MediaData * aSample)159 RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder<ConfigType>::Encode(
160     const MediaData* aSample) {
161   RefPtr<AndroidDataEncoder> self = this;
162   MOZ_ASSERT(aSample != nullptr);
163 
164   RefPtr<const MediaData> sample(aSample);
165   return InvokeAsync(mTaskQueue, __func__, [self, sample]() {
166     return self->ProcessEncode(std::move(sample));
167   });
168 }
169 
ConvertI420ToNV12Buffer(RefPtr<const VideoData> aSample,RefPtr<MediaByteBuffer> & aYUVBuffer)170 static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
171     RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer) {
172   const PlanarYCbCrImage* image = aSample->mImage->AsPlanarYCbCrImage();
173   MOZ_ASSERT(image);
174   const PlanarYCbCrData* yuv = image->GetData();
175   size_t ySize = yuv->mYStride * yuv->mYSize.height;
176   size_t size = ySize + (yuv->mCbCrStride * yuv->mCbCrSize.height * 2);
177   if (!aYUVBuffer || aYUVBuffer->Capacity() < size) {
178     aYUVBuffer = MakeRefPtr<MediaByteBuffer>(size);
179     aYUVBuffer->SetLength(size);
180   } else {
181     MOZ_ASSERT(aYUVBuffer->Length() >= size);
182   }
183 
184   if (libyuv::I420ToNV12(yuv->mYChannel, yuv->mYStride, yuv->mCbChannel,
185                          yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride,
186                          aYUVBuffer->Elements(), yuv->mYStride,
187                          aYUVBuffer->Elements() + ySize, yuv->mCbCrStride * 2,
188                          yuv->mYSize.width, yuv->mYSize.height) != 0) {
189     return nullptr;
190   }
191 
192   return jni::ByteBuffer::New(aYUVBuffer->Elements(), aYUVBuffer->Length());
193 }
194 
195 template <typename ConfigType>
196 RefPtr<MediaDataEncoder::EncodePromise>
ProcessEncode(RefPtr<const MediaData> aSample)197 AndroidDataEncoder<ConfigType>::ProcessEncode(RefPtr<const MediaData> aSample) {
198   AssertOnTaskQueue();
199 
200   REJECT_IF_ERROR();
201 
202   RefPtr<const VideoData> sample(aSample->As<const VideoData>());
203   MOZ_ASSERT(sample);
204 
205   jni::ByteBuffer::LocalRef buffer =
206       ConvertI420ToNV12Buffer(sample, mYUVBuffer);
207   if (!buffer) {
208     return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
209   }
210 
211   if (aSample->mKeyframe) {
212     mInputBufferInfo->Set(0, mYUVBuffer->Length(),
213                           aSample->mTime.ToMicroseconds(),
214                           java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME);
215   } else {
216     mInputBufferInfo->Set(0, mYUVBuffer->Length(),
217                           aSample->mTime.ToMicroseconds(), 0);
218   }
219 
220   mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr);
221 
222   if (mEncodedData.Length() > 0) {
223     EncodedData pending = std::move(mEncodedData);
224     return EncodePromise::CreateAndResolve(std::move(pending), __func__);
225   } else {
226     return EncodePromise::CreateAndResolve(EncodedData(), __func__);
227   }
228 }
229 
230 class AutoRelease final {
231  public:
AutoRelease(java::CodecProxy::Param aEncoder,java::Sample::Param aSample)232   AutoRelease(java::CodecProxy::Param aEncoder, java::Sample::Param aSample)
233       : mEncoder(aEncoder), mSample(aSample) {}
234 
~AutoRelease()235   ~AutoRelease() { mEncoder->ReleaseOutput(mSample, false); }
236 
237  private:
238   java::CodecProxy::GlobalRef mEncoder;
239   java::Sample::GlobalRef mSample;
240 };
241 
ExtractCodecConfig(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aAsAnnexB)242 static RefPtr<MediaByteBuffer> ExtractCodecConfig(
243     java::SampleBuffer::Param aBuffer, const int32_t aOffset,
244     const int32_t aSize, const bool aAsAnnexB) {
245   auto annexB = MakeRefPtr<MediaByteBuffer>(aSize);
246   annexB->SetLength(aSize);
247   jni::ByteBuffer::LocalRef dest =
248       jni::ByteBuffer::New(annexB->Elements(), aSize);
249   aBuffer->WriteToByteBuffer(dest, aOffset, aSize);
250   if (aAsAnnexB) {
251     return annexB;
252   }
253   // Convert to avcC.
254   nsTArray<AnnexB::NALEntry> paramSets;
255   AnnexB::ParseNALEntries(
256       Span<const uint8_t>(annexB->Elements(), annexB->Length()), paramSets);
257 
258   auto avcc = MakeRefPtr<MediaByteBuffer>();
259   AnnexB::NALEntry& sps = paramSets.ElementAt(0);
260   AnnexB::NALEntry& pps = paramSets.ElementAt(1);
261   const uint8_t* spsPtr = annexB->Elements() + sps.mOffset;
262   H264::WriteExtraData(
263       avcc, spsPtr[1], spsPtr[2], spsPtr[3],
264       Span<const uint8_t>(spsPtr, sps.mSize),
265       Span<const uint8_t>(annexB->Elements() + pps.mOffset, pps.mSize));
266   return avcc;
267 }
268 
269 template <typename ConfigType>
ProcessOutput(java::Sample::GlobalRef && aSample,java::SampleBuffer::GlobalRef && aBuffer)270 void AndroidDataEncoder<ConfigType>::ProcessOutput(
271     java::Sample::GlobalRef&& aSample,
272     java::SampleBuffer::GlobalRef&& aBuffer) {
273   if (!mTaskQueue->IsCurrentThreadIn()) {
274     nsresult rv =
275         mTaskQueue->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&,
276                                                java::SampleBuffer::GlobalRef&&>(
277             "AndroidDataEncoder::ProcessOutput", this,
278             &AndroidDataEncoder::ProcessOutput, std::move(aSample),
279             std::move(aBuffer)));
280     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
281     Unused << rv;
282     return;
283   }
284   AssertOnTaskQueue();
285 
286   if (!mJavaEncoder) {
287     return;
288   }
289 
290   AutoRelease releaseSample(mJavaEncoder, aSample);
291 
292   java::sdk::BufferInfo::LocalRef info = aSample->Info();
293   MOZ_ASSERT(info);
294 
295   int32_t flags;
296   bool ok = NS_SUCCEEDED(info->Flags(&flags));
297   bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
298 
299   int32_t offset;
300   ok &= NS_SUCCEEDED(info->Offset(&offset));
301 
302   int32_t size;
303   ok &= NS_SUCCEEDED(info->Size(&size));
304 
305   int64_t presentationTimeUs;
306   ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
307 
308   if (!ok) {
309     return;
310   }
311 
312   if (size > 0) {
313     if ((flags & java::sdk::MediaCodec::BUFFER_FLAG_CODEC_CONFIG) != 0) {
314       mConfigData = ExtractCodecConfig(aBuffer, offset, size,
315                                        mConfig.mUsage == Usage::Realtime);
316       return;
317     }
318     RefPtr<MediaRawData> output =
319         GetOutputData(aBuffer, offset, size,
320                       !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_KEY_FRAME));
321     output->mEOS = isEOS;
322     output->mTime = media::TimeUnit::FromMicroseconds(presentationTimeUs);
323     mEncodedData.AppendElement(std::move(output));
324   }
325 
326   if (isEOS) {
327     mDrainState = DrainState::DRAINED;
328   }
329   if (!mDrainPromise.IsEmpty()) {
330     EncodedData pending = std::move(mEncodedData);
331     mDrainPromise.Resolve(std::move(pending), __func__);
332   }
333 }
334 
335 template <typename ConfigType>
GetOutputData(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aIsKeyFrame)336 RefPtr<MediaRawData> AndroidDataEncoder<ConfigType>::GetOutputData(
337     java::SampleBuffer::Param aBuffer, const int32_t aOffset,
338     const int32_t aSize, const bool aIsKeyFrame) {
339   // Copy frame data from Java buffer.
340   auto output = MakeRefPtr<MediaRawData>();
341   UniquePtr<MediaRawDataWriter> writer(output->CreateWriter());
342   if (!writer->SetSize(aSize)) {
343     AND_ENC_LOGE("fail to allocate output buffer");
344     return nullptr;
345   }
346 
347   jni::ByteBuffer::LocalRef buf = jni::ByteBuffer::New(writer->Data(), aSize);
348   aBuffer->WriteToByteBuffer(buf, aOffset, aSize);
349   output->mKeyframe = aIsKeyFrame;
350 
351   return output;
352 }
353 
354 // AVC/H.264 frame can be in avcC or Annex B and needs extra convertion steps.
355 template <>
356 RefPtr<MediaRawData>
GetOutputData(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aIsKeyFrame)357 AndroidDataEncoder<MediaDataEncoder::H264Config>::GetOutputData(
358     java::SampleBuffer::Param aBuffer, const int32_t aOffset,
359     const int32_t aSize, const bool aIsKeyFrame) {
360   auto output = MakeRefPtr<MediaRawData>();
361 
362   size_t prependSize = 0;
363   RefPtr<MediaByteBuffer> avccHeader;
364   if (aIsKeyFrame && mConfigData) {
365     if (mConfig.mUsage == Usage::Realtime) {
366       prependSize = mConfigData->Length();
367     } else {
368       avccHeader = mConfigData;
369     }
370   }
371 
372   UniquePtr<MediaRawDataWriter> writer(output->CreateWriter());
373   if (!writer->SetSize(prependSize + aSize)) {
374     AND_ENC_LOGE("fail to allocate output buffer");
375     return nullptr;
376   }
377 
378   if (prependSize > 0) {
379     PodCopy(writer->Data(), mConfigData->Elements(), prependSize);
380   }
381 
382   jni::ByteBuffer::LocalRef buf =
383       jni::ByteBuffer::New(writer->Data() + prependSize, aSize);
384   aBuffer->WriteToByteBuffer(buf, aOffset, aSize);
385 
386   if (mConfig.mUsage != Usage::Realtime &&
387       !AnnexB::ConvertSampleToAVCC(output, avccHeader)) {
388     AND_ENC_LOGE("fail to convert annex-b sample to AVCC");
389     return nullptr;
390   }
391 
392   output->mKeyframe = aIsKeyFrame;
393 
394   return output;
395 }
396 
397 template <typename ConfigType>
398 RefPtr<MediaDataEncoder::EncodePromise>
Drain()399 AndroidDataEncoder<ConfigType>::Drain() {
400   return InvokeAsync(mTaskQueue, this, __func__,
401                      &AndroidDataEncoder<ConfigType>::ProcessDrain);
402 }
403 
404 template <typename ConfigType>
405 RefPtr<MediaDataEncoder::EncodePromise>
ProcessDrain()406 AndroidDataEncoder<ConfigType>::ProcessDrain() {
407   AssertOnTaskQueue();
408   MOZ_ASSERT(mJavaEncoder);
409   MOZ_ASSERT(mDrainPromise.IsEmpty());
410 
411   REJECT_IF_ERROR();
412 
413   switch (mDrainState) {
414     case DrainState::DRAINABLE:
415       mInputBufferInfo->Set(0, 0, -1,
416                             java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
417       mJavaEncoder->Input(nullptr, mInputBufferInfo, nullptr);
418       mDrainState = DrainState::DRAINING;
419       [[fallthrough]];
420     case DrainState::DRAINING:
421       if (mEncodedData.IsEmpty()) {
422         return mDrainPromise.Ensure(__func__);  // Pending promise.
423       }
424       [[fallthrough]];
425     case DrainState::DRAINED:
426       if (mEncodedData.Length() > 0) {
427         EncodedData pending = std::move(mEncodedData);
428         return EncodePromise::CreateAndResolve(std::move(pending), __func__);
429       } else {
430         return EncodePromise::CreateAndResolve(EncodedData(), __func__);
431       }
432   }
433 }
434 
435 template <typename ConfigType>
Shutdown()436 RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::Shutdown() {
437   return InvokeAsync(mTaskQueue, this, __func__,
438                      &AndroidDataEncoder<ConfigType>::ProcessShutdown);
439 }
440 
441 template <typename ConfigType>
ProcessShutdown()442 RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::ProcessShutdown() {
443   AssertOnTaskQueue();
444   if (mJavaEncoder) {
445     mJavaEncoder->Release();
446     mJavaEncoder = nullptr;
447   }
448 
449   if (mJavaCallbacks) {
450     JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
451     JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
452     mJavaCallbacks = nullptr;
453   }
454 
455   mFormat = nullptr;
456 
457   return ShutdownPromise::CreateAndResolve(true, __func__);
458 }
459 
460 template <typename ConfigType>
SetBitrate(const MediaDataEncoder::Rate aBitsPerSec)461 RefPtr<GenericPromise> AndroidDataEncoder<ConfigType>::SetBitrate(
462     const MediaDataEncoder::Rate aBitsPerSec) {
463   RefPtr<AndroidDataEncoder> self(this);
464   return InvokeAsync(mTaskQueue, __func__, [self, aBitsPerSec]() {
465     self->mJavaEncoder->SetBitrate(aBitsPerSec);
466     return GenericPromise::CreateAndResolve(true, __func__);
467   });
468 
469   return nullptr;
470 }
471 
472 template <typename ConfigType>
Error(const MediaResult & aError)473 void AndroidDataEncoder<ConfigType>::Error(const MediaResult& aError) {
474   if (!mTaskQueue->IsCurrentThreadIn()) {
475     nsresult rv = mTaskQueue->Dispatch(NewRunnableMethod<MediaResult>(
476         "AndroidDataEncoder::Error", this,
477         &AndroidDataEncoder<ConfigType>::Error, aError));
478     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
479     Unused << rv;
480     return;
481   }
482   AssertOnTaskQueue();
483 
484   mError = Some(aError);
485 }
486 
487 template <typename ConfigType>
HandleInput(int64_t aTimestamp,bool aProcessed)488 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleInput(
489     int64_t aTimestamp, bool aProcessed) {}
490 
491 template <typename ConfigType>
HandleOutput(java::Sample::Param aSample,java::SampleBuffer::Param aBuffer)492 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleOutput(
493     java::Sample::Param aSample, java::SampleBuffer::Param aBuffer) {
494   mEncoder->ProcessOutput(std::move(aSample), std::move(aBuffer));
495 }
496 
497 template <typename ConfigType>
498 void AndroidDataEncoder<ConfigType>::CallbacksSupport::
HandleOutputFormatChanged(java::sdk::MediaFormat::Param aFormat)499     HandleOutputFormatChanged(java::sdk::MediaFormat::Param aFormat) {}
500 
501 template <typename ConfigType>
HandleError(const MediaResult & aError)502 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleError(
503     const MediaResult& aError) {
504   mEncoder->Error(aError);
505 }
506 
507 // Force compiler to generate code.
508 template class AndroidDataEncoder<MediaDataEncoder::H264Config>;
509 template class AndroidDataEncoder<MediaDataEncoder::VP8Config>;
510 template class AndroidDataEncoder<MediaDataEncoder::VP9Config>;
511 }  // namespace mozilla
512 
513 #undef AND_ENC_LOG
514 #undef AND_ENC_LOGE
515