1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 #include "AndroidDataEncoder.h"
6
7 #include "AnnexB.h"
8 #include "MediaData.h"
9 #include "MediaInfo.h"
10 #include "SimpleMap.h"
11
12 #include "ImageContainer.h"
13 #include "mozilla/Logging.h"
14 #include "mozilla/ResultVariant.h"
15
16 #include "nsMimeTypes.h"
17
18 #include "libyuv.h"
19
20 namespace mozilla {
21 using media::TimeUnit;
22
23 extern LazyLogModule sPEMLog;
24 #define AND_ENC_LOG(arg, ...) \
25 MOZ_LOG(sPEMLog, mozilla::LogLevel::Debug, \
26 ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
27 #define AND_ENC_LOGE(arg, ...) \
28 MOZ_LOG(sPEMLog, mozilla::LogLevel::Error, \
29 ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
30
31 #define REJECT_IF_ERROR() \
32 do { \
33 if (mError) { \
34 auto error = mError.value(); \
35 mError.reset(); \
36 return EncodePromise::CreateAndReject(std::move(error), __func__); \
37 } \
38 } while (0)
39
40 template <typename ConfigType>
Init()41 RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder<ConfigType>::Init() {
42 // Sanity-check the input size for Android software encoder fails to do it.
43 if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
44 return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__);
45 }
46
47 return InvokeAsync(mTaskQueue, this, __func__,
48 &AndroidDataEncoder<ConfigType>::ProcessInit);
49 }
50
MimeTypeOf(MediaDataEncoder::CodecType aCodec)51 static const char* MimeTypeOf(MediaDataEncoder::CodecType aCodec) {
52 switch (aCodec) {
53 case MediaDataEncoder::CodecType::H264:
54 return "video/avc";
55 case MediaDataEncoder::CodecType::VP8:
56 return "video/x-vnd.on2.vp8";
57 case MediaDataEncoder::CodecType::VP9:
58 return "video/x-vnd.on2.vp9";
59 default:
60 return "";
61 }
62 }
63
64 using FormatResult = Result<java::sdk::MediaFormat::LocalRef, MediaResult>;
65
66 template <typename ConfigType>
ToMediaFormat(const ConfigType & aConfig)67 FormatResult ToMediaFormat(const ConfigType& aConfig) {
68 nsresult rv = NS_OK;
69 java::sdk::MediaFormat::LocalRef format;
70 rv = java::sdk::MediaFormat::CreateVideoFormat(MimeTypeOf(aConfig.mCodecType),
71 aConfig.mSize.width,
72 aConfig.mSize.height, &format);
73 NS_ENSURE_SUCCESS(
74 rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
75 "fail to create Java MediaFormat object")));
76
77 rv =
78 format->SetInteger(java::sdk::MediaFormat::KEY_BITRATE_MODE, 2 /* CBR */);
79 NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
80 "fail to set bitrate mode")));
81
82 rv = format->SetInteger(java::sdk::MediaFormat::KEY_BIT_RATE,
83 aConfig.mBitsPerSec);
84 NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
85 "fail to set bitrate")));
86
87 // COLOR_FormatYUV420SemiPlanar(NV12) is the most widely supported
88 // format.
89 rv = format->SetInteger(java::sdk::MediaFormat::KEY_COLOR_FORMAT, 0x15);
90 NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
91 "fail to set color format")));
92
93 rv = format->SetInteger(java::sdk::MediaFormat::KEY_FRAME_RATE,
94 aConfig.mFramerate);
95 NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
96 "fail to set frame rate")));
97
98 // Ensure interval >= 1. A negative value means no key frames are
99 // requested after the first frame. A zero value means a stream
100 // containing all key frames is requested.
101 int32_t intervalInSec =
102 std::max<size_t>(1, aConfig.mKeyframeInterval / aConfig.mFramerate);
103 rv = format->SetInteger(java::sdk::MediaFormat::KEY_I_FRAME_INTERVAL,
104 intervalInSec);
105 NS_ENSURE_SUCCESS(rv,
106 FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
107 "fail to set I-frame interval")));
108
109 return format;
110 }
111
112 template <typename ConfigType>
113 RefPtr<MediaDataEncoder::InitPromise>
ProcessInit()114 AndroidDataEncoder<ConfigType>::ProcessInit() {
115 AssertOnTaskQueue();
116 MOZ_ASSERT(!mJavaEncoder);
117
118 java::sdk::BufferInfo::LocalRef bufferInfo;
119 if (NS_FAILED(java::sdk::BufferInfo::New(&bufferInfo)) || !bufferInfo) {
120 return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
121 }
122 mInputBufferInfo = bufferInfo;
123
124 FormatResult result = ToMediaFormat<ConfigType>(mConfig);
125 if (result.isErr()) {
126 return InitPromise::CreateAndReject(result.unwrapErr(), __func__);
127 }
128 mFormat = result.unwrap();
129
130 // Register native methods.
131 JavaCallbacksSupport::Init();
132
133 mJavaCallbacks = java::CodecProxy::NativeCallbacks::New();
134 if (!mJavaCallbacks) {
135 return InitPromise::CreateAndReject(
136 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
137 "cannot create Java callback object"),
138 __func__);
139 }
140 JavaCallbacksSupport::AttachNative(
141 mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
142
143 mJavaEncoder = java::CodecProxy::Create(true /* encoder */, mFormat, nullptr,
144 mJavaCallbacks, u""_ns);
145 if (!mJavaEncoder) {
146 return InitPromise::CreateAndReject(
147 MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
148 "cannot create Java encoder object"),
149 __func__);
150 }
151
152 mIsHardwareAccelerated = mJavaEncoder->IsHardwareAccelerated();
153 mDrainState = DrainState::DRAINABLE;
154
155 return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
156 }
157
158 template <typename ConfigType>
Encode(const MediaData * aSample)159 RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder<ConfigType>::Encode(
160 const MediaData* aSample) {
161 RefPtr<AndroidDataEncoder> self = this;
162 MOZ_ASSERT(aSample != nullptr);
163
164 RefPtr<const MediaData> sample(aSample);
165 return InvokeAsync(mTaskQueue, __func__, [self, sample]() {
166 return self->ProcessEncode(std::move(sample));
167 });
168 }
169
ConvertI420ToNV12Buffer(RefPtr<const VideoData> aSample,RefPtr<MediaByteBuffer> & aYUVBuffer)170 static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
171 RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer) {
172 const PlanarYCbCrImage* image = aSample->mImage->AsPlanarYCbCrImage();
173 MOZ_ASSERT(image);
174 const PlanarYCbCrData* yuv = image->GetData();
175 size_t ySize = yuv->mYStride * yuv->mYSize.height;
176 size_t size = ySize + (yuv->mCbCrStride * yuv->mCbCrSize.height * 2);
177 if (!aYUVBuffer || aYUVBuffer->Capacity() < size) {
178 aYUVBuffer = MakeRefPtr<MediaByteBuffer>(size);
179 aYUVBuffer->SetLength(size);
180 } else {
181 MOZ_ASSERT(aYUVBuffer->Length() >= size);
182 }
183
184 if (libyuv::I420ToNV12(yuv->mYChannel, yuv->mYStride, yuv->mCbChannel,
185 yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride,
186 aYUVBuffer->Elements(), yuv->mYStride,
187 aYUVBuffer->Elements() + ySize, yuv->mCbCrStride * 2,
188 yuv->mYSize.width, yuv->mYSize.height) != 0) {
189 return nullptr;
190 }
191
192 return jni::ByteBuffer::New(aYUVBuffer->Elements(), aYUVBuffer->Length());
193 }
194
195 template <typename ConfigType>
196 RefPtr<MediaDataEncoder::EncodePromise>
ProcessEncode(RefPtr<const MediaData> aSample)197 AndroidDataEncoder<ConfigType>::ProcessEncode(RefPtr<const MediaData> aSample) {
198 AssertOnTaskQueue();
199
200 REJECT_IF_ERROR();
201
202 RefPtr<const VideoData> sample(aSample->As<const VideoData>());
203 MOZ_ASSERT(sample);
204
205 jni::ByteBuffer::LocalRef buffer =
206 ConvertI420ToNV12Buffer(sample, mYUVBuffer);
207 if (!buffer) {
208 return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
209 }
210
211 if (aSample->mKeyframe) {
212 mInputBufferInfo->Set(0, mYUVBuffer->Length(),
213 aSample->mTime.ToMicroseconds(),
214 java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME);
215 } else {
216 mInputBufferInfo->Set(0, mYUVBuffer->Length(),
217 aSample->mTime.ToMicroseconds(), 0);
218 }
219
220 mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr);
221
222 if (mEncodedData.Length() > 0) {
223 EncodedData pending = std::move(mEncodedData);
224 return EncodePromise::CreateAndResolve(std::move(pending), __func__);
225 } else {
226 return EncodePromise::CreateAndResolve(EncodedData(), __func__);
227 }
228 }
229
230 class AutoRelease final {
231 public:
AutoRelease(java::CodecProxy::Param aEncoder,java::Sample::Param aSample)232 AutoRelease(java::CodecProxy::Param aEncoder, java::Sample::Param aSample)
233 : mEncoder(aEncoder), mSample(aSample) {}
234
~AutoRelease()235 ~AutoRelease() { mEncoder->ReleaseOutput(mSample, false); }
236
237 private:
238 java::CodecProxy::GlobalRef mEncoder;
239 java::Sample::GlobalRef mSample;
240 };
241
ExtractCodecConfig(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aAsAnnexB)242 static RefPtr<MediaByteBuffer> ExtractCodecConfig(
243 java::SampleBuffer::Param aBuffer, const int32_t aOffset,
244 const int32_t aSize, const bool aAsAnnexB) {
245 auto annexB = MakeRefPtr<MediaByteBuffer>(aSize);
246 annexB->SetLength(aSize);
247 jni::ByteBuffer::LocalRef dest =
248 jni::ByteBuffer::New(annexB->Elements(), aSize);
249 aBuffer->WriteToByteBuffer(dest, aOffset, aSize);
250 if (aAsAnnexB) {
251 return annexB;
252 }
253 // Convert to avcC.
254 nsTArray<AnnexB::NALEntry> paramSets;
255 AnnexB::ParseNALEntries(
256 Span<const uint8_t>(annexB->Elements(), annexB->Length()), paramSets);
257
258 auto avcc = MakeRefPtr<MediaByteBuffer>();
259 AnnexB::NALEntry& sps = paramSets.ElementAt(0);
260 AnnexB::NALEntry& pps = paramSets.ElementAt(1);
261 const uint8_t* spsPtr = annexB->Elements() + sps.mOffset;
262 H264::WriteExtraData(
263 avcc, spsPtr[1], spsPtr[2], spsPtr[3],
264 Span<const uint8_t>(spsPtr, sps.mSize),
265 Span<const uint8_t>(annexB->Elements() + pps.mOffset, pps.mSize));
266 return avcc;
267 }
268
269 template <typename ConfigType>
ProcessOutput(java::Sample::GlobalRef && aSample,java::SampleBuffer::GlobalRef && aBuffer)270 void AndroidDataEncoder<ConfigType>::ProcessOutput(
271 java::Sample::GlobalRef&& aSample,
272 java::SampleBuffer::GlobalRef&& aBuffer) {
273 if (!mTaskQueue->IsCurrentThreadIn()) {
274 nsresult rv =
275 mTaskQueue->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&,
276 java::SampleBuffer::GlobalRef&&>(
277 "AndroidDataEncoder::ProcessOutput", this,
278 &AndroidDataEncoder::ProcessOutput, std::move(aSample),
279 std::move(aBuffer)));
280 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
281 Unused << rv;
282 return;
283 }
284 AssertOnTaskQueue();
285
286 if (!mJavaEncoder) {
287 return;
288 }
289
290 AutoRelease releaseSample(mJavaEncoder, aSample);
291
292 java::sdk::BufferInfo::LocalRef info = aSample->Info();
293 MOZ_ASSERT(info);
294
295 int32_t flags;
296 bool ok = NS_SUCCEEDED(info->Flags(&flags));
297 bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
298
299 int32_t offset;
300 ok &= NS_SUCCEEDED(info->Offset(&offset));
301
302 int32_t size;
303 ok &= NS_SUCCEEDED(info->Size(&size));
304
305 int64_t presentationTimeUs;
306 ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
307
308 if (!ok) {
309 return;
310 }
311
312 if (size > 0) {
313 if ((flags & java::sdk::MediaCodec::BUFFER_FLAG_CODEC_CONFIG) != 0) {
314 mConfigData = ExtractCodecConfig(aBuffer, offset, size,
315 mConfig.mUsage == Usage::Realtime);
316 return;
317 }
318 RefPtr<MediaRawData> output =
319 GetOutputData(aBuffer, offset, size,
320 !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_KEY_FRAME));
321 output->mEOS = isEOS;
322 output->mTime = media::TimeUnit::FromMicroseconds(presentationTimeUs);
323 mEncodedData.AppendElement(std::move(output));
324 }
325
326 if (isEOS) {
327 mDrainState = DrainState::DRAINED;
328 }
329 if (!mDrainPromise.IsEmpty()) {
330 EncodedData pending = std::move(mEncodedData);
331 mDrainPromise.Resolve(std::move(pending), __func__);
332 }
333 }
334
335 template <typename ConfigType>
GetOutputData(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aIsKeyFrame)336 RefPtr<MediaRawData> AndroidDataEncoder<ConfigType>::GetOutputData(
337 java::SampleBuffer::Param aBuffer, const int32_t aOffset,
338 const int32_t aSize, const bool aIsKeyFrame) {
339 // Copy frame data from Java buffer.
340 auto output = MakeRefPtr<MediaRawData>();
341 UniquePtr<MediaRawDataWriter> writer(output->CreateWriter());
342 if (!writer->SetSize(aSize)) {
343 AND_ENC_LOGE("fail to allocate output buffer");
344 return nullptr;
345 }
346
347 jni::ByteBuffer::LocalRef buf = jni::ByteBuffer::New(writer->Data(), aSize);
348 aBuffer->WriteToByteBuffer(buf, aOffset, aSize);
349 output->mKeyframe = aIsKeyFrame;
350
351 return output;
352 }
353
354 // AVC/H.264 frame can be in avcC or Annex B and needs extra convertion steps.
355 template <>
356 RefPtr<MediaRawData>
GetOutputData(java::SampleBuffer::Param aBuffer,const int32_t aOffset,const int32_t aSize,const bool aIsKeyFrame)357 AndroidDataEncoder<MediaDataEncoder::H264Config>::GetOutputData(
358 java::SampleBuffer::Param aBuffer, const int32_t aOffset,
359 const int32_t aSize, const bool aIsKeyFrame) {
360 auto output = MakeRefPtr<MediaRawData>();
361
362 size_t prependSize = 0;
363 RefPtr<MediaByteBuffer> avccHeader;
364 if (aIsKeyFrame && mConfigData) {
365 if (mConfig.mUsage == Usage::Realtime) {
366 prependSize = mConfigData->Length();
367 } else {
368 avccHeader = mConfigData;
369 }
370 }
371
372 UniquePtr<MediaRawDataWriter> writer(output->CreateWriter());
373 if (!writer->SetSize(prependSize + aSize)) {
374 AND_ENC_LOGE("fail to allocate output buffer");
375 return nullptr;
376 }
377
378 if (prependSize > 0) {
379 PodCopy(writer->Data(), mConfigData->Elements(), prependSize);
380 }
381
382 jni::ByteBuffer::LocalRef buf =
383 jni::ByteBuffer::New(writer->Data() + prependSize, aSize);
384 aBuffer->WriteToByteBuffer(buf, aOffset, aSize);
385
386 if (mConfig.mUsage != Usage::Realtime &&
387 !AnnexB::ConvertSampleToAVCC(output, avccHeader)) {
388 AND_ENC_LOGE("fail to convert annex-b sample to AVCC");
389 return nullptr;
390 }
391
392 output->mKeyframe = aIsKeyFrame;
393
394 return output;
395 }
396
397 template <typename ConfigType>
398 RefPtr<MediaDataEncoder::EncodePromise>
Drain()399 AndroidDataEncoder<ConfigType>::Drain() {
400 return InvokeAsync(mTaskQueue, this, __func__,
401 &AndroidDataEncoder<ConfigType>::ProcessDrain);
402 }
403
404 template <typename ConfigType>
405 RefPtr<MediaDataEncoder::EncodePromise>
ProcessDrain()406 AndroidDataEncoder<ConfigType>::ProcessDrain() {
407 AssertOnTaskQueue();
408 MOZ_ASSERT(mJavaEncoder);
409 MOZ_ASSERT(mDrainPromise.IsEmpty());
410
411 REJECT_IF_ERROR();
412
413 switch (mDrainState) {
414 case DrainState::DRAINABLE:
415 mInputBufferInfo->Set(0, 0, -1,
416 java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
417 mJavaEncoder->Input(nullptr, mInputBufferInfo, nullptr);
418 mDrainState = DrainState::DRAINING;
419 [[fallthrough]];
420 case DrainState::DRAINING:
421 if (mEncodedData.IsEmpty()) {
422 return mDrainPromise.Ensure(__func__); // Pending promise.
423 }
424 [[fallthrough]];
425 case DrainState::DRAINED:
426 if (mEncodedData.Length() > 0) {
427 EncodedData pending = std::move(mEncodedData);
428 return EncodePromise::CreateAndResolve(std::move(pending), __func__);
429 } else {
430 return EncodePromise::CreateAndResolve(EncodedData(), __func__);
431 }
432 }
433 }
434
435 template <typename ConfigType>
Shutdown()436 RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::Shutdown() {
437 return InvokeAsync(mTaskQueue, this, __func__,
438 &AndroidDataEncoder<ConfigType>::ProcessShutdown);
439 }
440
441 template <typename ConfigType>
ProcessShutdown()442 RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::ProcessShutdown() {
443 AssertOnTaskQueue();
444 if (mJavaEncoder) {
445 mJavaEncoder->Release();
446 mJavaEncoder = nullptr;
447 }
448
449 if (mJavaCallbacks) {
450 JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
451 JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
452 mJavaCallbacks = nullptr;
453 }
454
455 mFormat = nullptr;
456
457 return ShutdownPromise::CreateAndResolve(true, __func__);
458 }
459
460 template <typename ConfigType>
SetBitrate(const MediaDataEncoder::Rate aBitsPerSec)461 RefPtr<GenericPromise> AndroidDataEncoder<ConfigType>::SetBitrate(
462 const MediaDataEncoder::Rate aBitsPerSec) {
463 RefPtr<AndroidDataEncoder> self(this);
464 return InvokeAsync(mTaskQueue, __func__, [self, aBitsPerSec]() {
465 self->mJavaEncoder->SetBitrate(aBitsPerSec);
466 return GenericPromise::CreateAndResolve(true, __func__);
467 });
468
469 return nullptr;
470 }
471
472 template <typename ConfigType>
Error(const MediaResult & aError)473 void AndroidDataEncoder<ConfigType>::Error(const MediaResult& aError) {
474 if (!mTaskQueue->IsCurrentThreadIn()) {
475 nsresult rv = mTaskQueue->Dispatch(NewRunnableMethod<MediaResult>(
476 "AndroidDataEncoder::Error", this,
477 &AndroidDataEncoder<ConfigType>::Error, aError));
478 MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
479 Unused << rv;
480 return;
481 }
482 AssertOnTaskQueue();
483
484 mError = Some(aError);
485 }
486
487 template <typename ConfigType>
HandleInput(int64_t aTimestamp,bool aProcessed)488 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleInput(
489 int64_t aTimestamp, bool aProcessed) {}
490
491 template <typename ConfigType>
HandleOutput(java::Sample::Param aSample,java::SampleBuffer::Param aBuffer)492 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleOutput(
493 java::Sample::Param aSample, java::SampleBuffer::Param aBuffer) {
494 mEncoder->ProcessOutput(std::move(aSample), std::move(aBuffer));
495 }
496
497 template <typename ConfigType>
498 void AndroidDataEncoder<ConfigType>::CallbacksSupport::
HandleOutputFormatChanged(java::sdk::MediaFormat::Param aFormat)499 HandleOutputFormatChanged(java::sdk::MediaFormat::Param aFormat) {}
500
501 template <typename ConfigType>
HandleError(const MediaResult & aError)502 void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleError(
503 const MediaResult& aError) {
504 mEncoder->Error(aError);
505 }
506
507 // Force compiler to generate code.
508 template class AndroidDataEncoder<MediaDataEncoder::H264Config>;
509 template class AndroidDataEncoder<MediaDataEncoder::VP8Config>;
510 template class AndroidDataEncoder<MediaDataEncoder::VP9Config>;
511 } // namespace mozilla
512
513 #undef AND_ENC_LOG
514 #undef AND_ENC_LOGE
515