1 /* 2 * Copyright (c) 2012, The WebRTC project authors. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are 6 * met: 7 * 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 11 * * Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * * Neither the name of Google nor the names of its contributors may 17 * be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #ifndef WEBRTCGMPVIDEOCODEC_H_ 34 #define WEBRTCGMPVIDEOCODEC_H_ 35 36 #include <queue> 37 #include <string> 38 39 #include "nsThreadUtils.h" 40 #include "mozilla/Monitor.h" 41 #include "mozilla/Mutex.h" 42 43 #include "mozIGeckoMediaPluginService.h" 44 #include "MediaConduitInterface.h" 45 #include "AudioConduit.h" 46 #include "VideoConduit.h" 47 #include "api/video/video_frame_type.h" 48 #include "modules/video_coding/include/video_codec_interface.h" 49 50 #include "gmp-video-host.h" 51 #include "GMPVideoDecoderProxy.h" 52 #include "GMPVideoEncoderProxy.h" 53 54 #include "jsapi/PeerConnectionImpl.h" 55 56 namespace mozilla { 57 58 class GmpInitDoneRunnable : public Runnable { 59 public: GmpInitDoneRunnable(std::string aPCHandle)60 explicit GmpInitDoneRunnable(std::string aPCHandle) 61 : Runnable("GmpInitDoneRunnable"), 62 mResult(WEBRTC_VIDEO_CODEC_OK), 63 mPCHandle(std::move(aPCHandle)) {} 64 Run()65 NS_IMETHOD Run() override { 66 if (mResult == WEBRTC_VIDEO_CODEC_OK) { 67 // Might be useful to notify the PeerConnection about successful init 68 // someday. 69 return NS_OK; 70 } 71 72 PeerConnectionWrapper wrapper(mPCHandle); 73 if (wrapper.impl()) { 74 wrapper.impl()->OnMediaError(mError); 75 } 76 return NS_OK; 77 } 78 79 void Dispatch(int32_t aResult, const std::string& aError = "") { 80 mResult = aResult; 81 mError = aError; 82 nsCOMPtr<nsIThread> mainThread(do_GetMainThread()); 83 if (mainThread) { 84 // For some reason, the compiler on CI is treating |this| as a const 85 // pointer, despite the fact that we're in a non-const function. And, 86 // interestingly enough, correcting this doesn't require a const_cast. 87 mainThread->Dispatch(do_AddRef(static_cast<nsIRunnable*>(this)), 88 NS_DISPATCH_NORMAL); 89 } 90 } 91 Result()92 int32_t Result() { return mResult; } 93 94 private: 95 int32_t mResult; 96 const std::string mPCHandle; 97 std::string mError; 98 }; 99 100 // Hold a frame for later decode 101 class GMPDecodeData { 102 public: GMPDecodeData(const webrtc::EncodedImage & aInputImage,bool aMissingFrames,int64_t aRenderTimeMs)103 GMPDecodeData(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, 104 int64_t aRenderTimeMs) 105 : mImage(aInputImage), 106 mMissingFrames(aMissingFrames), 107 mRenderTimeMs(aRenderTimeMs) { 108 // We want to use this for queuing, and the calling code recycles the 109 // buffer on return from Decode() 110 MOZ_RELEASE_ASSERT(aInputImage.size() < 111 (std::numeric_limits<size_t>::max() >> 1)); 112 } 113 114 ~GMPDecodeData() = default; 115 116 const webrtc::EncodedImage mImage; 117 const bool mMissingFrames; 118 const int64_t mRenderTimeMs; 119 }; 120 121 class RefCountedWebrtcVideoEncoder { 122 public: 123 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RefCountedWebrtcVideoEncoder); 124 125 // Implement sort of WebrtcVideoEncoder interface and support refcounting. 126 // (We cannot use |Release|, since that's needed for nsRefPtr) 127 virtual int32_t InitEncode( 128 const webrtc::VideoCodec* aCodecSettings, 129 const webrtc::VideoEncoder::Settings& aSettings) = 0; 130 131 virtual int32_t Encode( 132 const webrtc::VideoFrame& aInputImage, 133 const std::vector<webrtc::VideoFrameType>* aFrameTypes) = 0; 134 135 virtual int32_t RegisterEncodeCompleteCallback( 136 webrtc::EncodedImageCallback* aCallback) = 0; 137 138 virtual int32_t Shutdown() = 0; 139 140 virtual int32_t SetRates( 141 const webrtc::VideoEncoder::RateControlParameters& aParameters) = 0; 142 143 virtual MediaEventSource<uint64_t>* InitPluginEvent() = 0; 144 145 virtual MediaEventSource<uint64_t>* ReleasePluginEvent() = 0; 146 147 protected: 148 virtual ~RefCountedWebrtcVideoEncoder() = default; 149 }; 150 151 class WebrtcGmpVideoEncoder : public GMPVideoEncoderCallbackProxy, 152 public RefCountedWebrtcVideoEncoder { 153 public: 154 explicit WebrtcGmpVideoEncoder(std::string aPCHandle); 155 156 // Implement VideoEncoder interface, sort of. 157 // (We cannot use |Release|, since that's needed for nsRefPtr) 158 int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, 159 const webrtc::VideoEncoder::Settings& aSettings) override; 160 161 int32_t Encode( 162 const webrtc::VideoFrame& aInputImage, 163 const std::vector<webrtc::VideoFrameType>* aFrameTypes) override; 164 165 int32_t RegisterEncodeCompleteCallback( 166 webrtc::EncodedImageCallback* aCallback) override; 167 168 int32_t Shutdown() override; 169 170 int32_t SetRates( 171 const webrtc::VideoEncoder::RateControlParameters& aParameters) override; 172 InitPluginEvent()173 MediaEventSource<uint64_t>* InitPluginEvent() override { 174 return &mInitPluginEvent; 175 } 176 ReleasePluginEvent()177 MediaEventSource<uint64_t>* ReleasePluginEvent() override { 178 return &mReleasePluginEvent; 179 } 180 181 // GMPVideoEncoderCallback virtual functions. 182 virtual void Terminated() override; 183 184 virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame, 185 const nsTArray<uint8_t>& aCodecSpecificInfo) override; 186 Error(GMPErr aError)187 virtual void Error(GMPErr aError) override {} 188 189 private: 190 virtual ~WebrtcGmpVideoEncoder(); 191 192 static void InitEncode_g(const RefPtr<WebrtcGmpVideoEncoder>& aThis, 193 const GMPVideoCodec& aCodecParams, 194 int32_t aNumberOfCores, uint32_t aMaxPayloadSize, 195 const RefPtr<GmpInitDoneRunnable>& aInitDone); 196 int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, 197 const GMPVideoCodec& aCodecParams, 198 std::string* aErrorOut); 199 int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, 200 std::string* aErrorOut); 201 int32_t InitEncoderForSize(unsigned short aWidth, unsigned short aHeight, 202 std::string* aErrorOut); 203 static void ReleaseGmp_g(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder); 204 void Close_g(); 205 206 class InitDoneCallback : public GetGMPVideoEncoderCallback { 207 public: InitDoneCallback(const RefPtr<WebrtcGmpVideoEncoder> & aEncoder,const RefPtr<GmpInitDoneRunnable> & aInitDone,const GMPVideoCodec & aCodecParams)208 InitDoneCallback(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder, 209 const RefPtr<GmpInitDoneRunnable>& aInitDone, 210 const GMPVideoCodec& aCodecParams) 211 : mEncoder(aEncoder), 212 mInitDone(aInitDone), 213 mCodecParams(aCodecParams) {} 214 Done(GMPVideoEncoderProxy * aGMP,GMPVideoHost * aHost)215 virtual void Done(GMPVideoEncoderProxy* aGMP, 216 GMPVideoHost* aHost) override { 217 std::string errorOut; 218 int32_t result = 219 mEncoder->GmpInitDone(aGMP, aHost, mCodecParams, &errorOut); 220 221 mInitDone->Dispatch(result, errorOut); 222 } 223 224 private: 225 const RefPtr<WebrtcGmpVideoEncoder> mEncoder; 226 const RefPtr<GmpInitDoneRunnable> mInitDone; 227 const GMPVideoCodec mCodecParams; 228 }; 229 230 static void Encode_g(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder, 231 webrtc::VideoFrame aInputImage, 232 std::vector<webrtc::VideoFrameType> aFrameTypes); 233 void RegetEncoderForResolutionChange( 234 uint32_t aWidth, uint32_t aHeight, 235 const RefPtr<GmpInitDoneRunnable>& aInitDone); 236 237 class InitDoneForResolutionChangeCallback 238 : public GetGMPVideoEncoderCallback { 239 public: InitDoneForResolutionChangeCallback(const RefPtr<WebrtcGmpVideoEncoder> & aEncoder,const RefPtr<GmpInitDoneRunnable> & aInitDone,uint32_t aWidth,uint32_t aHeight)240 InitDoneForResolutionChangeCallback( 241 const RefPtr<WebrtcGmpVideoEncoder>& aEncoder, 242 const RefPtr<GmpInitDoneRunnable>& aInitDone, uint32_t aWidth, 243 uint32_t aHeight) 244 : mEncoder(aEncoder), 245 mInitDone(aInitDone), 246 mWidth(aWidth), 247 mHeight(aHeight) {} 248 Done(GMPVideoEncoderProxy * aGMP,GMPVideoHost * aHost)249 virtual void Done(GMPVideoEncoderProxy* aGMP, 250 GMPVideoHost* aHost) override { 251 std::string errorOut; 252 int32_t result = mEncoder->GmpInitDone(aGMP, aHost, &errorOut); 253 if (result != WEBRTC_VIDEO_CODEC_OK) { 254 mInitDone->Dispatch(result, errorOut); 255 return; 256 } 257 258 result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut); 259 mInitDone->Dispatch(result, errorOut); 260 } 261 262 private: 263 const RefPtr<WebrtcGmpVideoEncoder> mEncoder; 264 const RefPtr<GmpInitDoneRunnable> mInitDone; 265 const uint32_t mWidth; 266 const uint32_t mHeight; 267 }; 268 269 static int32_t SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis, 270 uint32_t aNewBitRateKbps, Maybe<double> aFrameRate); 271 272 nsCOMPtr<mozIGeckoMediaPluginService> mMPS; 273 nsCOMPtr<nsIThread> mGMPThread; 274 GMPVideoEncoderProxy* mGMP; 275 // Used to handle a race where Release() is called while init is in progress 276 bool mInitting; 277 GMPVideoHost* mHost; 278 GMPVideoCodec mCodecParams; 279 uint32_t mMaxPayloadSize; 280 webrtc::CodecSpecificInfo mCodecSpecificInfo; 281 // Protects mCallback 282 Mutex mCallbackMutex; 283 webrtc::EncodedImageCallback* mCallback; 284 Maybe<uint64_t> mCachedPluginId; 285 const std::string mPCHandle; 286 287 struct InputImageData { 288 int64_t timestamp_us; 289 }; 290 // Map rtp time -> input image data 291 DataMutex<std::map<uint32_t, InputImageData>> mInputImageMap; 292 293 MediaEventProducer<uint64_t> mInitPluginEvent; 294 MediaEventProducer<uint64_t> mReleasePluginEvent; 295 }; 296 297 // Basically a strong ref to a RefCountedWebrtcVideoEncoder, that also 298 // translates from Release() to RefCountedWebrtcVideoEncoder::Shutdown(), 299 // since we need RefCountedWebrtcVideoEncoder::Release() for managing the 300 // refcount. The webrtc.org code gets one of these, so it doesn't unilaterally 301 // delete the "real" encoder. 302 class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder { 303 public: WebrtcVideoEncoderProxy(RefPtr<RefCountedWebrtcVideoEncoder> aEncoder)304 explicit WebrtcVideoEncoderProxy( 305 RefPtr<RefCountedWebrtcVideoEncoder> aEncoder) 306 : mEncoderImpl(std::move(aEncoder)) {} 307 ~WebrtcVideoEncoderProxy()308 virtual ~WebrtcVideoEncoderProxy() { 309 RegisterEncodeCompleteCallback(nullptr); 310 } 311 InitPluginEvent()312 MediaEventSource<uint64_t>* InitPluginEvent() override { 313 return mEncoderImpl->InitPluginEvent(); 314 } 315 ReleasePluginEvent()316 MediaEventSource<uint64_t>* ReleasePluginEvent() override { 317 return mEncoderImpl->ReleasePluginEvent(); 318 } 319 InitEncode(const webrtc::VideoCodec * aCodecSettings,const WebrtcVideoEncoder::Settings & aSettings)320 int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, 321 const WebrtcVideoEncoder::Settings& aSettings) override { 322 return mEncoderImpl->InitEncode(aCodecSettings, aSettings); 323 } 324 Encode(const webrtc::VideoFrame & aInputImage,const std::vector<webrtc::VideoFrameType> * aFrameTypes)325 int32_t Encode( 326 const webrtc::VideoFrame& aInputImage, 327 const std::vector<webrtc::VideoFrameType>* aFrameTypes) override { 328 return mEncoderImpl->Encode(aInputImage, aFrameTypes); 329 } 330 RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * aCallback)331 int32_t RegisterEncodeCompleteCallback( 332 webrtc::EncodedImageCallback* aCallback) override { 333 return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback); 334 } 335 Release()336 int32_t Release() override { return mEncoderImpl->Shutdown(); } 337 SetRates(const RateControlParameters & aParameters)338 void SetRates(const RateControlParameters& aParameters) override { 339 mEncoderImpl->SetRates(aParameters); 340 } 341 342 private: 343 const RefPtr<RefCountedWebrtcVideoEncoder> mEncoderImpl; 344 }; 345 346 class WebrtcGmpVideoDecoder : public GMPVideoDecoderCallbackProxy { 347 public: 348 explicit WebrtcGmpVideoDecoder(std::string aPCHandle); 349 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder); 350 351 // Implement VideoEncoder interface, sort of. 352 // (We cannot use |Release|, since that's needed for nsRefPtr) 353 virtual int32_t InitDecode(const webrtc::VideoCodec* aCodecSettings, 354 int32_t aNumberOfCores); 355 virtual int32_t Decode(const webrtc::EncodedImage& aInputImage, 356 bool aMissingFrames, int64_t aRenderTimeMs); 357 virtual int32_t RegisterDecodeCompleteCallback( 358 webrtc::DecodedImageCallback* aCallback); 359 360 virtual int32_t ReleaseGmp(); 361 InitPluginEvent()362 MediaEventSource<uint64_t>* InitPluginEvent() { return &mInitPluginEvent; } 363 ReleasePluginEvent()364 MediaEventSource<uint64_t>* ReleasePluginEvent() { 365 return &mReleasePluginEvent; 366 } 367 368 // GMPVideoDecoderCallbackProxy 369 virtual void Terminated() override; 370 371 virtual void Decoded(GMPVideoi420Frame* aDecodedFrame) override; 372 ReceivedDecodedReferenceFrame(const uint64_t aPictureId)373 virtual void ReceivedDecodedReferenceFrame( 374 const uint64_t aPictureId) override { 375 MOZ_CRASH(); 376 } 377 ReceivedDecodedFrame(const uint64_t aPictureId)378 virtual void ReceivedDecodedFrame(const uint64_t aPictureId) override { 379 MOZ_CRASH(); 380 } 381 InputDataExhausted()382 virtual void InputDataExhausted() override {} 383 DrainComplete()384 virtual void DrainComplete() override {} 385 ResetComplete()386 virtual void ResetComplete() override {} 387 Error(GMPErr aError)388 virtual void Error(GMPErr aError) override { mDecoderStatus = aError; } 389 390 private: 391 virtual ~WebrtcGmpVideoDecoder(); 392 393 static void InitDecode_g(const RefPtr<WebrtcGmpVideoDecoder>& aThis, 394 const webrtc::VideoCodec* aCodecSettings, 395 int32_t aNumberOfCores, 396 const RefPtr<GmpInitDoneRunnable>& aInitDone); 397 int32_t GmpInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost, 398 std::string* aErrorOut); 399 static void ReleaseGmp_g(const RefPtr<WebrtcGmpVideoDecoder>& aDecoder); 400 void Close_g(); 401 402 class InitDoneCallback : public GetGMPVideoDecoderCallback { 403 public: InitDoneCallback(const RefPtr<WebrtcGmpVideoDecoder> & aDecoder,const RefPtr<GmpInitDoneRunnable> & aInitDone)404 explicit InitDoneCallback(const RefPtr<WebrtcGmpVideoDecoder>& aDecoder, 405 const RefPtr<GmpInitDoneRunnable>& aInitDone) 406 : mDecoder(aDecoder), mInitDone(aInitDone) {} 407 Done(GMPVideoDecoderProxy * aGMP,GMPVideoHost * aHost)408 virtual void Done(GMPVideoDecoderProxy* aGMP, 409 GMPVideoHost* aHost) override { 410 std::string errorOut; 411 int32_t result = mDecoder->GmpInitDone(aGMP, aHost, &errorOut); 412 413 mInitDone->Dispatch(result, errorOut); 414 } 415 416 private: 417 const RefPtr<WebrtcGmpVideoDecoder> mDecoder; 418 const RefPtr<GmpInitDoneRunnable> mInitDone; 419 }; 420 421 static void Decode_g(const RefPtr<WebrtcGmpVideoDecoder>& aThis, 422 UniquePtr<GMPDecodeData>&& aDecodeData); 423 424 nsCOMPtr<mozIGeckoMediaPluginService> mMPS; 425 nsCOMPtr<nsIThread> mGMPThread; 426 GMPVideoDecoderProxy* mGMP; // Addref is held for us 427 // Used to handle a race where Release() is called while init is in progress 428 bool mInitting; 429 // Frames queued for decode while mInitting is true 430 nsTArray<UniquePtr<GMPDecodeData>> mQueuedFrames; 431 GMPVideoHost* mHost; 432 // Protects mCallback 433 Mutex mCallbackMutex; 434 webrtc::DecodedImageCallback* mCallback; 435 Maybe<uint64_t> mCachedPluginId; 436 Atomic<GMPErr, ReleaseAcquire> mDecoderStatus; 437 const std::string mPCHandle; 438 439 MediaEventProducer<uint64_t> mInitPluginEvent; 440 MediaEventProducer<uint64_t> mReleasePluginEvent; 441 }; 442 443 // Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates 444 // from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need 445 // WebrtcGmpVideoDecoder::Release() for managing the refcount. 446 // The webrtc.org code gets one of these, so it doesn't unilaterally delete 447 // the "real" encoder. 448 class WebrtcVideoDecoderProxy : public WebrtcVideoDecoder { 449 public: WebrtcVideoDecoderProxy(std::string aPCHandle)450 explicit WebrtcVideoDecoderProxy(std::string aPCHandle) 451 : mDecoderImpl(new WebrtcGmpVideoDecoder(std::move(aPCHandle))) {} 452 ~WebrtcVideoDecoderProxy()453 virtual ~WebrtcVideoDecoderProxy() { 454 RegisterDecodeCompleteCallback(nullptr); 455 } 456 InitPluginEvent()457 MediaEventSource<uint64_t>* InitPluginEvent() override { 458 return mDecoderImpl->InitPluginEvent(); 459 } 460 ReleasePluginEvent()461 MediaEventSource<uint64_t>* ReleasePluginEvent() override { 462 return mDecoderImpl->ReleasePluginEvent(); 463 } 464 InitDecode(const webrtc::VideoCodec * aCodecSettings,int32_t aNumberOfCores)465 int32_t InitDecode(const webrtc::VideoCodec* aCodecSettings, 466 int32_t aNumberOfCores) override { 467 return mDecoderImpl->InitDecode(aCodecSettings, aNumberOfCores); 468 } 469 Decode(const webrtc::EncodedImage & aInputImage,bool aMissingFrames,int64_t aRenderTimeMs)470 int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, 471 int64_t aRenderTimeMs) override { 472 return mDecoderImpl->Decode(aInputImage, aMissingFrames, aRenderTimeMs); 473 } 474 RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * aCallback)475 int32_t RegisterDecodeCompleteCallback( 476 webrtc::DecodedImageCallback* aCallback) override { 477 return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback); 478 } 479 Release()480 int32_t Release() override { return mDecoderImpl->ReleaseGmp(); } 481 482 private: 483 const RefPtr<WebrtcGmpVideoDecoder> mDecoderImpl; 484 }; 485 486 } // namespace mozilla 487 488 #endif 489