1 /* This Source Code Form is subject to the terms of the Mozilla Public
2  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3  * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 
5 #include "WebrtcGmpVideoCodec.h"
6 
7 #include <iostream>
8 #include <vector>
9 
10 #include "mozilla/Move.h"
11 #include "mozilla/SyncRunnable.h"
12 #include "VideoConduit.h"
13 #include "AudioConduit.h"
14 #include "runnable_utils.h"
15 
16 #include "mozIGeckoMediaPluginService.h"
17 #include "nsServiceManagerUtils.h"
18 #include "GMPVideoDecoderProxy.h"
19 #include "GMPVideoEncoderProxy.h"
20 #include "MainThreadUtils.h"
21 
22 #include "gmp-video-host.h"
23 #include "gmp-video-frame-i420.h"
24 #include "gmp-video-frame-encoded.h"
25 
26 #include "webrtc/video_engine/include/vie_external_codec.h"
27 
28 namespace mozilla {
29 
30 #ifdef LOG
31 #undef LOG
32 #endif
33 
34 #ifdef MOZILLA_INTERNAL_API
35 extern mozilla::LogModule* GetGMPLog();
36 #else
37 // For CPP unit tests
38 PRLogModuleInfo*
39 GetGMPLog()
40 {
41   static PRLogModuleInfo *sLog;
42   if (!sLog)
43     sLog = PR_NewLogModule("GMP");
44   return sLog;
45 }
46 #endif
47 #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg)
48 #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg)
49 
WebrtcGmpPCHandleSetter(const std::string & aPCHandle)50 WebrtcGmpPCHandleSetter::WebrtcGmpPCHandleSetter(const std::string& aPCHandle)
51 {
52   if (!NS_IsMainThread()) {
53     MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
54     return;
55   }
56   MOZ_ASSERT(sCurrentHandle.empty());
57   sCurrentHandle = aPCHandle;
58 }
59 
~WebrtcGmpPCHandleSetter()60 WebrtcGmpPCHandleSetter::~WebrtcGmpPCHandleSetter()
61 {
62   if (!NS_IsMainThread()) {
63     MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
64     return;
65   }
66 
67   sCurrentHandle.clear();
68 }
69 
70 /* static */ std::string
GetCurrentHandle()71 WebrtcGmpPCHandleSetter::GetCurrentHandle()
72 {
73   if (!NS_IsMainThread()) {
74     MOZ_ASSERT(false, "WebrtcGmpPCHandleSetter can only be used on main");
75     return "";
76   }
77 
78   return sCurrentHandle;
79 }
80 
81 std::string WebrtcGmpPCHandleSetter::sCurrentHandle = "";
82 
83 // Encoder.
WebrtcGmpVideoEncoder()84 WebrtcGmpVideoEncoder::WebrtcGmpVideoEncoder()
85   : mGMP(nullptr)
86   , mInitting(false)
87   , mHost(nullptr)
88   , mMaxPayloadSize(0)
89   , mCallbackMutex("WebrtcGmpVideoEncoder encoded callback mutex")
90   , mCallback(nullptr)
91   , mCachedPluginId(0)
92 {
93 #ifdef MOZILLA_INTERNAL_API
94   if (mPCHandle.empty()) {
95     mPCHandle = WebrtcGmpPCHandleSetter::GetCurrentHandle();
96   }
97   MOZ_ASSERT(!mPCHandle.empty());
98 #endif
99 }
100 
~WebrtcGmpVideoEncoder()101 WebrtcGmpVideoEncoder::~WebrtcGmpVideoEncoder()
102 {
103   // We should not have been destroyed if we never closed our GMP
104   MOZ_ASSERT(!mGMP);
105 }
106 
107 static int
WebrtcFrameTypeToGmpFrameType(webrtc::VideoFrameType aIn,GMPVideoFrameType * aOut)108 WebrtcFrameTypeToGmpFrameType(webrtc::VideoFrameType aIn,
109                               GMPVideoFrameType *aOut)
110 {
111   MOZ_ASSERT(aOut);
112   switch(aIn) {
113     case webrtc::kKeyFrame:
114       *aOut = kGMPKeyFrame;
115       break;
116     case webrtc::kDeltaFrame:
117       *aOut = kGMPDeltaFrame;
118       break;
119     case webrtc::kGoldenFrame:
120       *aOut = kGMPGoldenFrame;
121       break;
122     case webrtc::kAltRefFrame:
123       *aOut = kGMPAltRefFrame;
124       break;
125     case webrtc::kSkipFrame:
126       *aOut = kGMPSkipFrame;
127       break;
128     default:
129       MOZ_CRASH("Unexpected VideoFrameType");
130   }
131 
132   return WEBRTC_VIDEO_CODEC_OK;
133 }
134 
135 static int
GmpFrameTypeToWebrtcFrameType(GMPVideoFrameType aIn,webrtc::VideoFrameType * aOut)136 GmpFrameTypeToWebrtcFrameType(GMPVideoFrameType aIn,
137                               webrtc::VideoFrameType *aOut)
138 {
139   MOZ_ASSERT(aOut);
140   switch(aIn) {
141     case kGMPKeyFrame:
142       *aOut = webrtc::kKeyFrame;
143       break;
144     case kGMPDeltaFrame:
145       *aOut = webrtc::kDeltaFrame;
146       break;
147     case kGMPGoldenFrame:
148       *aOut = webrtc::kGoldenFrame;
149       break;
150     case kGMPAltRefFrame:
151       *aOut = webrtc::kAltRefFrame;
152       break;
153     case kGMPSkipFrame:
154       *aOut = webrtc::kSkipFrame;
155       break;
156     default:
157       MOZ_CRASH("Unexpected GMPVideoFrameType");
158   }
159 
160   return WEBRTC_VIDEO_CODEC_OK;
161 }
162 
163 int32_t
InitEncode(const webrtc::VideoCodec * aCodecSettings,int32_t aNumberOfCores,uint32_t aMaxPayloadSize)164 WebrtcGmpVideoEncoder::InitEncode(const webrtc::VideoCodec* aCodecSettings,
165                                   int32_t aNumberOfCores,
166                                   uint32_t aMaxPayloadSize)
167 {
168   if (!mMPS) {
169     mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
170   }
171   MOZ_ASSERT(mMPS);
172 
173   if (!mGMPThread) {
174     if (NS_WARN_IF(NS_FAILED(mMPS->GetThread(getter_AddRefs(mGMPThread))))) {
175       return WEBRTC_VIDEO_CODEC_ERROR;
176     }
177   }
178 
179   // Bug XXXXXX: transfer settings from codecSettings to codec.
180   GMPVideoCodec codecParams;
181   memset(&codecParams, 0, sizeof(codecParams));
182 
183   codecParams.mGMPApiVersion = 33;
184   codecParams.mStartBitrate = aCodecSettings->startBitrate;
185   codecParams.mMinBitrate = aCodecSettings->minBitrate;
186   codecParams.mMaxBitrate = aCodecSettings->maxBitrate;
187   codecParams.mMaxFramerate = aCodecSettings->maxFramerate;
188   mMaxPayloadSize = aMaxPayloadSize;
189 
190   memset(&mCodecSpecificInfo, 0, sizeof(webrtc::CodecSpecificInfo));
191   mCodecSpecificInfo.codecType = webrtc::kVideoCodecH264;
192   mCodecSpecificInfo.codecSpecific.H264.packetizationMode = aCodecSettings->codecSpecific.H264.packetizationMode;
193   if (mCodecSpecificInfo.codecSpecific.H264.packetizationMode == 1) {
194     mMaxPayloadSize = 0; // No limit.
195   }
196 
197   if (aCodecSettings->mode == webrtc::kScreensharing) {
198     codecParams.mMode = kGMPScreensharing;
199   } else {
200     codecParams.mMode = kGMPRealtimeVideo;
201   }
202 
203   codecParams.mWidth = aCodecSettings->width;
204   codecParams.mHeight = aCodecSettings->height;
205 
206   RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
207   mGMPThread->Dispatch(WrapRunnableNM(WebrtcGmpVideoEncoder::InitEncode_g,
208                                       RefPtr<WebrtcGmpVideoEncoder>(this),
209                                       codecParams,
210                                       aNumberOfCores,
211                                       aMaxPayloadSize,
212                                       initDone),
213                        NS_DISPATCH_NORMAL);
214 
215   // Since init of the GMP encoder is a multi-step async dispatch (including
216   // dispatches to main), and since this function is invoked on main, there's
217   // no safe way to block until this init is done. If an error occurs, we'll
218   // handle it later.
219   return WEBRTC_VIDEO_CODEC_OK;
220 }
221 
222 /* static */
223 void
InitEncode_g(const RefPtr<WebrtcGmpVideoEncoder> & aThis,const GMPVideoCodec & aCodecParams,int32_t aNumberOfCores,uint32_t aMaxPayloadSize,const RefPtr<GmpInitDoneRunnable> & aInitDone)224 WebrtcGmpVideoEncoder::InitEncode_g(
225     const RefPtr<WebrtcGmpVideoEncoder>& aThis,
226     const GMPVideoCodec& aCodecParams,
227     int32_t aNumberOfCores,
228     uint32_t aMaxPayloadSize,
229     const RefPtr<GmpInitDoneRunnable>& aInitDone)
230 {
231   nsTArray<nsCString> tags;
232   tags.AppendElement(NS_LITERAL_CSTRING("h264"));
233   UniquePtr<GetGMPVideoEncoderCallback> callback(
234     new InitDoneCallback(aThis, aInitDone, aCodecParams, aMaxPayloadSize));
235   aThis->mInitting = true;
236   nsresult rv = aThis->mMPS->GetGMPVideoEncoder(nullptr,
237                                                 &tags,
238                                                 NS_LITERAL_CSTRING(""),
239                                                 Move(callback));
240   if (NS_WARN_IF(NS_FAILED(rv))) {
241     LOGD(("GMP Encode: GetGMPVideoEncoder failed"));
242     aThis->Close_g();
243     aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
244                         "GMP Encode: GetGMPVideoEncoder failed");
245   }
246 }
247 
248 int32_t
GmpInitDone(GMPVideoEncoderProxy * aGMP,GMPVideoHost * aHost,std::string * aErrorOut)249 WebrtcGmpVideoEncoder::GmpInitDone(GMPVideoEncoderProxy* aGMP,
250                                    GMPVideoHost* aHost,
251                                    std::string* aErrorOut)
252 {
253   if (!mInitting || !aGMP || !aHost) {
254     *aErrorOut = "GMP Encode: Either init was aborted, "
255                  "or init failed to supply either a GMP Encoder or GMP host.";
256     if (aGMP) {
257       // This could destroy us, since aGMP may be the last thing holding a ref
258       // Return immediately.
259       aGMP->Close();
260     }
261     return WEBRTC_VIDEO_CODEC_ERROR;
262   }
263 
264   mInitting = false;
265 
266   if (mGMP && mGMP != aGMP) {
267     Close_g();
268   }
269 
270   mGMP = aGMP;
271   mHost = aHost;
272   mCachedPluginId = mGMP->GetPluginId();
273   return WEBRTC_VIDEO_CODEC_OK;
274 }
275 
276 int32_t
GmpInitDone(GMPVideoEncoderProxy * aGMP,GMPVideoHost * aHost,const GMPVideoCodec & aCodecParams,uint32_t aMaxPayloadSize,std::string * aErrorOut)277 WebrtcGmpVideoEncoder::GmpInitDone(GMPVideoEncoderProxy* aGMP,
278                                    GMPVideoHost* aHost,
279                                    const GMPVideoCodec& aCodecParams,
280                                    uint32_t aMaxPayloadSize,
281                                    std::string* aErrorOut)
282 {
283   int32_t r = GmpInitDone(aGMP, aHost, aErrorOut);
284   if (r != WEBRTC_VIDEO_CODEC_OK) {
285     // We might have been destroyed if GmpInitDone failed.
286     // Return immediately.
287     return r;
288   }
289   mCodecParams = aCodecParams;
290   return InitEncoderForSize(aCodecParams.mWidth,
291                             aCodecParams.mHeight,
292                             aErrorOut);
293 }
294 
295 void
Close_g()296 WebrtcGmpVideoEncoder::Close_g()
297 {
298   GMPVideoEncoderProxy* gmp(mGMP);
299   mGMP = nullptr;
300   mHost = nullptr;
301   mInitting = false;
302 
303   if (gmp) {
304     // Do this last, since this could cause us to be destroyed
305     gmp->Close();
306   }
307 }
308 
309 int32_t
InitEncoderForSize(unsigned short aWidth,unsigned short aHeight,std::string * aErrorOut)310 WebrtcGmpVideoEncoder::InitEncoderForSize(unsigned short aWidth,
311                                           unsigned short aHeight,
312                                           std::string* aErrorOut)
313 {
314   mCodecParams.mWidth = aWidth;
315   mCodecParams.mHeight = aHeight;
316   // Pass dummy codecSpecific data for now...
317   nsTArray<uint8_t> codecSpecific;
318 
319   GMPErr err = mGMP->InitEncode(mCodecParams, codecSpecific, this, 1, mMaxPayloadSize);
320   if (err != GMPNoErr) {
321     *aErrorOut = "GMP Encode: InitEncode failed";
322     return WEBRTC_VIDEO_CODEC_ERROR;
323   }
324 
325   return WEBRTC_VIDEO_CODEC_OK;
326 }
327 
328 
329 int32_t
Encode(const webrtc::I420VideoFrame & aInputImage,const webrtc::CodecSpecificInfo * aCodecSpecificInfo,const std::vector<webrtc::VideoFrameType> * aFrameTypes)330 WebrtcGmpVideoEncoder::Encode(const webrtc::I420VideoFrame& aInputImage,
331                               const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
332                               const std::vector<webrtc::VideoFrameType>* aFrameTypes)
333 {
334   MOZ_ASSERT(aInputImage.width() >= 0 && aInputImage.height() >= 0);
335   // Would be really nice to avoid this sync dispatch, but it would require a
336   // copy of the frame, since it doesn't appear to actually have a refcount.
337   mGMPThread->Dispatch(
338       WrapRunnable(this,
339                    &WebrtcGmpVideoEncoder::Encode_g,
340                    &aInputImage,
341                    aCodecSpecificInfo,
342                    aFrameTypes),
343       NS_DISPATCH_SYNC);
344 
345   return WEBRTC_VIDEO_CODEC_OK;
346 }
347 
348 void
RegetEncoderForResolutionChange(uint32_t aWidth,uint32_t aHeight,const RefPtr<GmpInitDoneRunnable> & aInitDone)349 WebrtcGmpVideoEncoder::RegetEncoderForResolutionChange(
350     uint32_t aWidth,
351     uint32_t aHeight,
352     const RefPtr<GmpInitDoneRunnable>& aInitDone)
353 {
354   Close_g();
355 
356   UniquePtr<GetGMPVideoEncoderCallback> callback(
357     new InitDoneForResolutionChangeCallback(this,
358                                             aInitDone,
359                                             aWidth,
360                                             aHeight));
361 
362   // OpenH264 codec (at least) can't handle dynamic input resolution changes
363   // re-init the plugin when the resolution changes
364   // XXX allow codec to indicate it doesn't need re-init!
365   nsTArray<nsCString> tags;
366   tags.AppendElement(NS_LITERAL_CSTRING("h264"));
367   mInitting = true;
368   if (NS_WARN_IF(NS_FAILED(mMPS->GetGMPVideoEncoder(nullptr,
369                                                     &tags,
370                                                     NS_LITERAL_CSTRING(""),
371                                                     Move(callback))))) {
372     aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
373                         "GMP Encode: GetGMPVideoEncoder failed");
374   }
375 }
376 
377 int32_t
Encode_g(const webrtc::I420VideoFrame * aInputImage,const webrtc::CodecSpecificInfo * aCodecSpecificInfo,const std::vector<webrtc::VideoFrameType> * aFrameTypes)378 WebrtcGmpVideoEncoder::Encode_g(const webrtc::I420VideoFrame* aInputImage,
379                                 const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
380                                 const std::vector<webrtc::VideoFrameType>* aFrameTypes)
381 {
382   if (!mGMP) {
383     // destroyed via Terminate(), failed to init, or just not initted yet
384     LOGD(("GMP Encode: not initted yet"));
385     return WEBRTC_VIDEO_CODEC_ERROR;
386   }
387   MOZ_ASSERT(mHost);
388 
389   if (static_cast<uint32_t>(aInputImage->width()) != mCodecParams.mWidth ||
390       static_cast<uint32_t>(aInputImage->height()) != mCodecParams.mHeight) {
391     LOGD(("GMP Encode: resolution change from %ux%u to %dx%d",
392           mCodecParams.mWidth, mCodecParams.mHeight, aInputImage->width(), aInputImage->height()));
393 
394     RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
395     RegetEncoderForResolutionChange(aInputImage->width(),
396                                     aInputImage->height(),
397                                     initDone);
398     if (!mGMP) {
399       // We needed to go async to re-get the encoder. Bail.
400       return WEBRTC_VIDEO_CODEC_ERROR;
401     }
402   }
403 
404   GMPVideoFrame* ftmp = nullptr;
405   GMPErr err = mHost->CreateFrame(kGMPI420VideoFrame, &ftmp);
406   if (err != GMPNoErr) {
407     return WEBRTC_VIDEO_CODEC_ERROR;
408   }
409   GMPUniquePtr<GMPVideoi420Frame> frame(static_cast<GMPVideoi420Frame*>(ftmp));
410 
411   err = frame->CreateFrame(aInputImage->allocated_size(webrtc::kYPlane),
412                            aInputImage->buffer(webrtc::kYPlane),
413                            aInputImage->allocated_size(webrtc::kUPlane),
414                            aInputImage->buffer(webrtc::kUPlane),
415                            aInputImage->allocated_size(webrtc::kVPlane),
416                            aInputImage->buffer(webrtc::kVPlane),
417                            aInputImage->width(),
418                            aInputImage->height(),
419                            aInputImage->stride(webrtc::kYPlane),
420                            aInputImage->stride(webrtc::kUPlane),
421                            aInputImage->stride(webrtc::kVPlane));
422   if (err != GMPNoErr) {
423     return err;
424   }
425   frame->SetTimestamp((aInputImage->timestamp() * 1000ll)/90); // note: rounds down!
426   //frame->SetDuration(1000000ll/30); // XXX base duration on measured current FPS - or don't bother
427 
428   // Bug XXXXXX: Set codecSpecific info
429   GMPCodecSpecificInfo info;
430   memset(&info, 0, sizeof(info));
431   info.mCodecType = kGMPVideoCodecH264;
432   nsTArray<uint8_t> codecSpecificInfo;
433   codecSpecificInfo.AppendElements((uint8_t*)&info, sizeof(GMPCodecSpecificInfo));
434 
435   nsTArray<GMPVideoFrameType> gmp_frame_types;
436   for (auto it = aFrameTypes->begin(); it != aFrameTypes->end(); ++it) {
437     GMPVideoFrameType ft;
438 
439     int32_t ret = WebrtcFrameTypeToGmpFrameType(*it, &ft);
440     if (ret != WEBRTC_VIDEO_CODEC_OK) {
441       return ret;
442     }
443 
444     gmp_frame_types.AppendElement(ft);
445   }
446 
447   LOGD(("GMP Encode: %llu", (aInputImage->timestamp() * 1000ll)/90));
448   err = mGMP->Encode(Move(frame), codecSpecificInfo, gmp_frame_types);
449   if (err != GMPNoErr) {
450     return err;
451   }
452 
453   return WEBRTC_VIDEO_CODEC_OK;
454 }
455 
456 int32_t
RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * aCallback)457 WebrtcGmpVideoEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* aCallback)
458 {
459   MutexAutoLock lock(mCallbackMutex);
460   mCallback = aCallback;
461 
462   return WEBRTC_VIDEO_CODEC_OK;
463 }
464 
465 /* static */ void
ReleaseGmp_g(RefPtr<WebrtcGmpVideoEncoder> & aEncoder)466 WebrtcGmpVideoEncoder::ReleaseGmp_g(RefPtr<WebrtcGmpVideoEncoder>& aEncoder)
467 {
468   aEncoder->Close_g();
469 }
470 
471 int32_t
ReleaseGmp()472 WebrtcGmpVideoEncoder::ReleaseGmp()
473 {
474   LOGD(("GMP Released:"));
475   if (mGMPThread) {
476     mGMPThread->Dispatch(
477         WrapRunnableNM(&WebrtcGmpVideoEncoder::ReleaseGmp_g,
478                        RefPtr<WebrtcGmpVideoEncoder>(this)),
479         NS_DISPATCH_NORMAL);
480   }
481   return WEBRTC_VIDEO_CODEC_OK;
482 }
483 
484 int32_t
SetChannelParameters(uint32_t aPacketLoss,int aRTT)485 WebrtcGmpVideoEncoder::SetChannelParameters(uint32_t aPacketLoss, int aRTT)
486 {
487   return WEBRTC_VIDEO_CODEC_OK;
488 }
489 
490 int32_t
SetRates(uint32_t aNewBitRate,uint32_t aFrameRate)491 WebrtcGmpVideoEncoder::SetRates(uint32_t aNewBitRate, uint32_t aFrameRate)
492 {
493   MOZ_ASSERT(mGMPThread);
494   if (aFrameRate == 0) {
495     aFrameRate = 30; // Assume 30fps if we don't know the rate
496   }
497   mGMPThread->Dispatch(WrapRunnableNM(&WebrtcGmpVideoEncoder::SetRates_g,
498                                       RefPtr<WebrtcGmpVideoEncoder>(this),
499                                       aNewBitRate,
500                                       aFrameRate),
501                        NS_DISPATCH_NORMAL);
502 
503   return WEBRTC_VIDEO_CODEC_OK;
504 }
505 
506 /* static */ int32_t
SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis,uint32_t aNewBitRate,uint32_t aFrameRate)507 WebrtcGmpVideoEncoder::SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis,
508                                   uint32_t aNewBitRate,
509                                   uint32_t aFrameRate)
510 {
511   if (!aThis->mGMP) {
512     // destroyed via Terminate()
513     return WEBRTC_VIDEO_CODEC_ERROR;
514   }
515 
516   GMPErr err = aThis->mGMP->SetRates(aNewBitRate, aFrameRate);
517   if (err != GMPNoErr) {
518     return WEBRTC_VIDEO_CODEC_ERROR;
519   }
520 
521   return WEBRTC_VIDEO_CODEC_OK;
522 }
523 
524 // GMPVideoEncoderCallback virtual functions.
525 void
Terminated()526 WebrtcGmpVideoEncoder::Terminated()
527 {
528   LOGD(("GMP Encoder Terminated: %p", (void *)this));
529 
530   mGMP->Close();
531   mGMP = nullptr;
532   mHost = nullptr;
533   mInitting = false;
534   // Could now notify that it's dead
535 }
536 
537 void
Encoded(GMPVideoEncodedFrame * aEncodedFrame,const nsTArray<uint8_t> & aCodecSpecificInfo)538 WebrtcGmpVideoEncoder::Encoded(GMPVideoEncodedFrame* aEncodedFrame,
539                                const nsTArray<uint8_t>& aCodecSpecificInfo)
540 {
541   MutexAutoLock lock(mCallbackMutex);
542   if (mCallback) {
543     webrtc::VideoFrameType ft;
544     GmpFrameTypeToWebrtcFrameType(aEncodedFrame->FrameType(), &ft);
545     uint32_t timestamp = (aEncodedFrame->TimeStamp() * 90ll + 999)/1000;
546 
547     LOGD(("GMP Encoded: %llu, type %d, len %d",
548          aEncodedFrame->TimeStamp(),
549          aEncodedFrame->BufferType(),
550          aEncodedFrame->Size()));
551 
552     // Right now makes one Encoded() callback per unit
553     // XXX convert to FragmentationHeader format (array of offsets and sizes plus a buffer) in
554     // combination with H264 packetization changes in webrtc/trunk code
555     uint8_t *buffer = aEncodedFrame->Buffer();
556     uint8_t *end = aEncodedFrame->Buffer() + aEncodedFrame->Size();
557     size_t size_bytes;
558     switch (aEncodedFrame->BufferType()) {
559       case GMP_BufferSingle:
560         size_bytes = 0;
561         break;
562       case GMP_BufferLength8:
563         size_bytes = 1;
564         break;
565       case GMP_BufferLength16:
566         size_bytes = 2;
567         break;
568       case GMP_BufferLength24:
569         size_bytes = 3;
570         break;
571       case GMP_BufferLength32:
572         size_bytes = 4;
573         break;
574       default:
575         // Really that it's not in the enum
576         LOG(LogLevel::Error,
577             ("GMP plugin returned incorrect type (%d)", aEncodedFrame->BufferType()));
578         // XXX Bug 1041232 - need a better API for interfacing to the
579         // plugin so we can kill it here
580         return;
581     }
582 
583     struct nal_entry {
584       uint32_t offset;
585       uint32_t size;
586     };
587     AutoTArray<nal_entry, 1> nals;
588     uint32_t size;
589     // make sure we don't read past the end of the buffer getting the size
590     while (buffer+size_bytes < end) {
591       switch (aEncodedFrame->BufferType()) {
592         case GMP_BufferSingle:
593           size = aEncodedFrame->Size();
594           break;
595         case GMP_BufferLength8:
596           size = *buffer++;
597           break;
598         case GMP_BufferLength16:
599           // presumes we can do unaligned loads
600           size = *(reinterpret_cast<uint16_t*>(buffer));
601           buffer += 2;
602           break;
603         case GMP_BufferLength24:
604           // 24-bits is a pain, since byte-order issues make things painful
605           // I'm going to define 24-bit as little-endian always; big-endian must convert
606           size = ((uint32_t) *buffer) |
607                  (((uint32_t) *(buffer+1)) << 8) |
608                  (((uint32_t) *(buffer+2)) << 16);
609           buffer += 3;
610           break;
611         case GMP_BufferLength32:
612           // presumes we can do unaligned loads
613           size = *(reinterpret_cast<uint32_t*>(buffer));
614           buffer += 4;
615           break;
616         default:
617           MOZ_CRASH("GMP_BufferType already handled in switch above");
618       }
619       if (buffer+size > end) {
620         // XXX see above - should we kill the plugin for returning extra bytes?  Probably
621         LOG(LogLevel::Error,
622             ("GMP plugin returned badly formatted encoded data: end is %td bytes past buffer end",
623              buffer+size - end));
624         return;
625       }
626       // XXX optimize by making buffer an offset
627       nal_entry nal = {((uint32_t) (buffer-aEncodedFrame->Buffer())), (uint32_t) size};
628       nals.AppendElement(nal);
629       buffer += size;
630       // on last one, buffer == end normally
631     }
632     if (buffer != end) {
633       // At most 3 bytes can be left over, depending on buffertype
634       LOGD(("GMP plugin returned %td extra bytes", end - buffer));
635     }
636 
637     size_t num_nals = nals.Length();
638     if (num_nals > 0) {
639       webrtc::RTPFragmentationHeader fragmentation;
640       fragmentation.VerifyAndAllocateFragmentationHeader(num_nals);
641       for (size_t i = 0; i < num_nals; i++) {
642         fragmentation.fragmentationOffset[i] = nals[i].offset;
643         fragmentation.fragmentationLength[i] = nals[i].size;
644       }
645 
646       webrtc::EncodedImage unit(aEncodedFrame->Buffer(), size, size);
647       unit._frameType = ft;
648       unit._timeStamp = timestamp;
649       // Ensure we ignore this when calculating RTCP timestamps
650       unit.capture_time_ms_ = -1;
651       unit._completeFrame = true;
652 
653       // TODO: Currently the OpenH264 codec does not preserve any codec
654       //       specific info passed into it and just returns default values.
655       //       If this changes in the future, it would be nice to get rid of
656       //       mCodecSpecificInfo.
657       mCallback->Encoded(unit, &mCodecSpecificInfo, &fragmentation);
658     }
659   }
660 }
661 
662 // Decoder.
WebrtcGmpVideoDecoder()663 WebrtcGmpVideoDecoder::WebrtcGmpVideoDecoder() :
664   mGMP(nullptr),
665   mInitting(false),
666   mHost(nullptr),
667   mCallbackMutex("WebrtcGmpVideoDecoder decoded callback mutex"),
668   mCallback(nullptr),
669   mCachedPluginId(0),
670   mDecoderStatus(GMPNoErr)
671 {
672 #ifdef MOZILLA_INTERNAL_API
673   if (mPCHandle.empty()) {
674     mPCHandle = WebrtcGmpPCHandleSetter::GetCurrentHandle();
675   }
676   MOZ_ASSERT(!mPCHandle.empty());
677 #endif
678 }
679 
~WebrtcGmpVideoDecoder()680 WebrtcGmpVideoDecoder::~WebrtcGmpVideoDecoder()
681 {
682   // We should not have been destroyed if we never closed our GMP
683   MOZ_ASSERT(!mGMP);
684 }
685 
686 int32_t
InitDecode(const webrtc::VideoCodec * aCodecSettings,int32_t aNumberOfCores)687 WebrtcGmpVideoDecoder::InitDecode(const webrtc::VideoCodec* aCodecSettings,
688                                   int32_t aNumberOfCores)
689 {
690   if (!mMPS) {
691     mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
692   }
693   MOZ_ASSERT(mMPS);
694 
695   if (!mGMPThread) {
696     if (NS_WARN_IF(NS_FAILED(mMPS->GetThread(getter_AddRefs(mGMPThread))))) {
697       return WEBRTC_VIDEO_CODEC_ERROR;
698     }
699   }
700 
701   RefPtr<GmpInitDoneRunnable> initDone(new GmpInitDoneRunnable(mPCHandle));
702   mGMPThread->Dispatch(WrapRunnableNM(&WebrtcGmpVideoDecoder::InitDecode_g,
703                                       RefPtr<WebrtcGmpVideoDecoder>(this),
704                                       aCodecSettings,
705                                       aNumberOfCores,
706                                       initDone),
707                        NS_DISPATCH_NORMAL);
708 
709   return WEBRTC_VIDEO_CODEC_OK;
710 }
711 
712 /* static */ void
InitDecode_g(const RefPtr<WebrtcGmpVideoDecoder> & aThis,const webrtc::VideoCodec * aCodecSettings,int32_t aNumberOfCores,const RefPtr<GmpInitDoneRunnable> & aInitDone)713 WebrtcGmpVideoDecoder::InitDecode_g(
714     const RefPtr<WebrtcGmpVideoDecoder>& aThis,
715     const webrtc::VideoCodec* aCodecSettings,
716     int32_t aNumberOfCores,
717     const RefPtr<GmpInitDoneRunnable>& aInitDone)
718 {
719   nsTArray<nsCString> tags;
720   tags.AppendElement(NS_LITERAL_CSTRING("h264"));
721   UniquePtr<GetGMPVideoDecoderCallback> callback(
722     new InitDoneCallback(aThis, aInitDone));
723   aThis->mInitting = true;
724   nsresult rv = aThis->mMPS->GetGMPVideoDecoder(nullptr,
725                                                 &tags,
726                                                 NS_LITERAL_CSTRING(""),
727                                                 Move(callback));
728   if (NS_WARN_IF(NS_FAILED(rv))) {
729     LOGD(("GMP Decode: GetGMPVideoDecoder failed"));
730     aThis->Close_g();
731     aInitDone->Dispatch(WEBRTC_VIDEO_CODEC_ERROR,
732                         "GMP Decode: GetGMPVideoDecoder failed.");
733   }
734 }
735 
736 int32_t
GmpInitDone(GMPVideoDecoderProxy * aGMP,GMPVideoHost * aHost,std::string * aErrorOut)737 WebrtcGmpVideoDecoder::GmpInitDone(GMPVideoDecoderProxy* aGMP,
738                                    GMPVideoHost* aHost,
739                                    std::string* aErrorOut)
740 {
741   if (!mInitting || !aGMP || !aHost) {
742     *aErrorOut = "GMP Decode: Either init was aborted, "
743                  "or init failed to supply either a GMP decoder or GMP host.";
744     if (aGMP) {
745       // This could destroy us, since aGMP may be the last thing holding a ref
746       // Return immediately.
747       aGMP->Close();
748     }
749     return WEBRTC_VIDEO_CODEC_ERROR;
750   }
751 
752   mInitting = false;
753 
754   if (mGMP && mGMP != aGMP) {
755     Close_g();
756   }
757 
758   mGMP = aGMP;
759   mHost = aHost;
760   mCachedPluginId = mGMP->GetPluginId();
761   // Bug XXXXXX: transfer settings from codecSettings to codec.
762   GMPVideoCodec codec;
763   memset(&codec, 0, sizeof(codec));
764   codec.mGMPApiVersion = 33;
765 
766   // XXX this is currently a hack
767   //GMPVideoCodecUnion codecSpecific;
768   //memset(&codecSpecific, 0, sizeof(codecSpecific));
769   nsTArray<uint8_t> codecSpecific;
770   nsresult rv = mGMP->InitDecode(codec, codecSpecific, this, 1);
771   if (NS_FAILED(rv)) {
772     *aErrorOut = "GMP Decode: InitDecode failed";
773     return WEBRTC_VIDEO_CODEC_ERROR;
774   }
775 
776   return WEBRTC_VIDEO_CODEC_OK;
777 }
778 
779 void
Close_g()780 WebrtcGmpVideoDecoder::Close_g()
781 {
782   GMPVideoDecoderProxy* gmp(mGMP);
783   mGMP = nullptr;
784   mHost = nullptr;
785   mInitting = false;
786 
787   if (gmp) {
788     // Do this last, since this could cause us to be destroyed
789     gmp->Close();
790   }
791 }
792 
793 int32_t
Decode(const webrtc::EncodedImage & aInputImage,bool aMissingFrames,const webrtc::RTPFragmentationHeader * aFragmentation,const webrtc::CodecSpecificInfo * aCodecSpecificInfo,int64_t aRenderTimeMs)794 WebrtcGmpVideoDecoder::Decode(const webrtc::EncodedImage& aInputImage,
795                               bool aMissingFrames,
796                               const webrtc::RTPFragmentationHeader* aFragmentation,
797                               const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
798                               int64_t aRenderTimeMs)
799 {
800   int32_t ret;
801   MOZ_ASSERT(mGMPThread);
802   MOZ_ASSERT(!NS_IsMainThread());
803   // Would be really nice to avoid this sync dispatch, but it would require a
804   // copy of the frame, since it doesn't appear to actually have a refcount.
805   mozilla::SyncRunnable::DispatchToThread(mGMPThread,
806                 WrapRunnableRet(&ret, this,
807                                 &WebrtcGmpVideoDecoder::Decode_g,
808                                 aInputImage,
809                                 aMissingFrames,
810                                 aFragmentation,
811                                 aCodecSpecificInfo,
812                                 aRenderTimeMs));
813 
814   return ret;
815 }
816 
817 int32_t
Decode_g(const webrtc::EncodedImage & aInputImage,bool aMissingFrames,const webrtc::RTPFragmentationHeader * aFragmentation,const webrtc::CodecSpecificInfo * aCodecSpecificInfo,int64_t aRenderTimeMs)818 WebrtcGmpVideoDecoder::Decode_g(const webrtc::EncodedImage& aInputImage,
819                                 bool aMissingFrames,
820                                 const webrtc::RTPFragmentationHeader* aFragmentation,
821                                 const webrtc::CodecSpecificInfo* aCodecSpecificInfo,
822                                 int64_t aRenderTimeMs)
823 {
824   if (!mGMP) {
825     // destroyed via Terminate(), failed to init, or just not initted yet
826     LOGD(("GMP Decode: not initted yet"));
827     return WEBRTC_VIDEO_CODEC_ERROR;
828   }
829   MOZ_ASSERT(mHost);
830 
831   if (!aInputImage._length) {
832     return WEBRTC_VIDEO_CODEC_ERROR;
833   }
834 
835   GMPVideoFrame* ftmp = nullptr;
836   GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
837   if (err != GMPNoErr) {
838     return WEBRTC_VIDEO_CODEC_ERROR;
839   }
840 
841   GMPUniquePtr<GMPVideoEncodedFrame> frame(static_cast<GMPVideoEncodedFrame*>(ftmp));
842   err = frame->CreateEmptyFrame(aInputImage._length);
843   if (err != GMPNoErr) {
844     return WEBRTC_VIDEO_CODEC_ERROR;
845   }
846 
847   // XXX At this point, we only will get mode1 data (a single length and a buffer)
848   // Session_info.cc/etc code needs to change to support mode 0.
849   *(reinterpret_cast<uint32_t*>(frame->Buffer())) = frame->Size();
850 
851   // XXX It'd be wonderful not to have to memcpy the encoded data!
852   memcpy(frame->Buffer()+4, aInputImage._buffer+4, frame->Size()-4);
853 
854   frame->SetEncodedWidth(aInputImage._encodedWidth);
855   frame->SetEncodedHeight(aInputImage._encodedHeight);
856   frame->SetTimeStamp((aInputImage._timeStamp * 1000ll)/90); // rounds down
857   frame->SetCompleteFrame(aInputImage._completeFrame);
858   frame->SetBufferType(GMP_BufferLength32);
859 
860   GMPVideoFrameType ft;
861   int32_t ret = WebrtcFrameTypeToGmpFrameType(aInputImage._frameType, &ft);
862   if (ret != WEBRTC_VIDEO_CODEC_OK) {
863     return ret;
864   }
865 
866   // Bug XXXXXX: Set codecSpecific info
867   GMPCodecSpecificInfo info;
868   memset(&info, 0, sizeof(info));
869   info.mCodecType = kGMPVideoCodecH264;
870   info.mCodecSpecific.mH264.mSimulcastIdx = 0;
871   nsTArray<uint8_t> codecSpecificInfo;
872   codecSpecificInfo.AppendElements((uint8_t*)&info, sizeof(GMPCodecSpecificInfo));
873 
874   LOGD(("GMP Decode: %llu, len %d", frame->TimeStamp(), aInputImage._length));
875   nsresult rv = mGMP->Decode(Move(frame),
876                              aMissingFrames,
877                              codecSpecificInfo,
878                              aRenderTimeMs);
879   if (NS_FAILED(rv)) {
880     return WEBRTC_VIDEO_CODEC_ERROR;
881   }
882   if(mDecoderStatus != GMPNoErr){
883     mDecoderStatus = GMPNoErr;
884     return WEBRTC_VIDEO_CODEC_ERROR;
885   }
886   return WEBRTC_VIDEO_CODEC_OK;
887 }
888 
889 int32_t
RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * aCallback)890 WebrtcGmpVideoDecoder::RegisterDecodeCompleteCallback( webrtc::DecodedImageCallback* aCallback)
891 {
892   MutexAutoLock lock(mCallbackMutex);
893   mCallback = aCallback;
894 
895   return WEBRTC_VIDEO_CODEC_OK;
896 }
897 
898 
899 /* static */ void
ReleaseGmp_g(RefPtr<WebrtcGmpVideoDecoder> & aDecoder)900 WebrtcGmpVideoDecoder::ReleaseGmp_g(RefPtr<WebrtcGmpVideoDecoder>& aDecoder)
901 {
902   aDecoder->Close_g();
903 }
904 
905 int32_t
ReleaseGmp()906 WebrtcGmpVideoDecoder::ReleaseGmp()
907 {
908   LOGD(("GMP Released:"));
909   RegisterDecodeCompleteCallback(nullptr);
910 
911   if (mGMPThread) {
912     mGMPThread->Dispatch(
913         WrapRunnableNM(&WebrtcGmpVideoDecoder::ReleaseGmp_g,
914                        RefPtr<WebrtcGmpVideoDecoder>(this)),
915         NS_DISPATCH_NORMAL);
916   }
917   return WEBRTC_VIDEO_CODEC_OK;
918 }
919 
920 int32_t
Reset()921 WebrtcGmpVideoDecoder::Reset()
922 {
923   // XXX ?
924   return WEBRTC_VIDEO_CODEC_OK;
925 }
926 
927 void
Terminated()928 WebrtcGmpVideoDecoder::Terminated()
929 {
930   LOGD(("GMP Decoder Terminated: %p", (void *)this));
931 
932   mGMP->Close();
933   mGMP = nullptr;
934   mHost = nullptr;
935   mInitting = false;
936   // Could now notify that it's dead
937 }
938 
939 void
Decoded(GMPVideoi420Frame * aDecodedFrame)940 WebrtcGmpVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame)
941 {
942   MutexAutoLock lock(mCallbackMutex);
943   if (mCallback) {
944     webrtc::I420VideoFrame image;
945     int ret = image.CreateFrame(aDecodedFrame->Buffer(kGMPYPlane),
946                                 aDecodedFrame->Buffer(kGMPUPlane),
947                                 aDecodedFrame->Buffer(kGMPVPlane),
948                                 aDecodedFrame->Width(),
949                                 aDecodedFrame->Height(),
950                                 aDecodedFrame->Stride(kGMPYPlane),
951                                 aDecodedFrame->Stride(kGMPUPlane),
952                                 aDecodedFrame->Stride(kGMPVPlane));
953     if (ret != 0) {
954       return;
955     }
956     image.set_timestamp((aDecodedFrame->Timestamp() * 90ll + 999)/1000); // round up
957     image.set_render_time_ms(0);
958 
959     LOGD(("GMP Decoded: %llu", aDecodedFrame->Timestamp()));
960     mCallback->Decoded(image);
961   }
962   aDecodedFrame->Destroy();
963 }
964 
965 }
966