1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 * You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifndef VIDEO_SESSION_H_ 6 #define VIDEO_SESSION_H_ 7 8 #include "nsAutoPtr.h" 9 #include "mozilla/Attributes.h" 10 #include "mozilla/Atomics.h" 11 12 #include "MediaConduitInterface.h" 13 #include "MediaEngineWrapper.h" 14 #include "CodecStatistics.h" 15 #include "LoadManagerFactory.h" 16 #include "LoadManager.h" 17 #include "runnable_utils.h" 18 19 // conflicts with #include of scoped_ptr.h 20 #undef FF 21 // Video Engine Includes 22 #include "webrtc/common_types.h" 23 #ifdef FF 24 #undef FF // Avoid name collision between scoped_ptr.h and nsCRTGlue.h. 25 #endif 26 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 27 #include "webrtc/video_engine/include/vie_base.h" 28 #include "webrtc/video_engine/include/vie_capture.h" 29 #include "webrtc/video_engine/include/vie_codec.h" 30 #include "webrtc/video_engine/include/vie_external_codec.h" 31 #include "webrtc/video_engine/include/vie_render.h" 32 #include "webrtc/video_engine/include/vie_network.h" 33 #include "webrtc/video_engine/include/vie_rtp_rtcp.h" 34 35 /** This file hosts several structures identifying different aspects 36 * of a RTP Session. 37 */ 38 39 using webrtc::ViEBase; 40 using webrtc::ViENetwork; 41 using webrtc::ViECodec; 42 using webrtc::ViECapture; 43 using webrtc::ViERender; 44 using webrtc::ViEExternalCapture; 45 using webrtc::ViEExternalCodec; 46 47 namespace mozilla { 48 49 class WebrtcAudioConduit; 50 class nsThread; 51 52 // Interface of external video encoder for WebRTC. 53 class WebrtcVideoEncoder:public VideoEncoder 54 ,public webrtc::VideoEncoder 55 {}; 56 57 // Interface of external video decoder for WebRTC. 58 class WebrtcVideoDecoder:public VideoDecoder 59 ,public webrtc::VideoDecoder 60 {}; 61 62 /** 63 * Concrete class for Video session. Hooks up 64 * - media-source and target to external transport 65 */ 66 class WebrtcVideoConduit : public VideoSessionConduit 67 , public webrtc::Transport 68 , public webrtc::ExternalRenderer 69 { 70 public: 71 //VoiceEngine defined constant for Payload Name Size. 72 static const unsigned int CODEC_PLNAME_SIZE; 73 74 /** 75 * Set up A/V sync between this (incoming) VideoConduit and an audio conduit. 76 */ 77 void SyncTo(WebrtcAudioConduit *aConduit); 78 79 /** 80 * Function to attach Renderer end-point for the Media-Video conduit. 81 * @param aRenderer : Reference to the concrete Video renderer implementation 82 * Note: Multiple invocations of this API shall remove an existing renderer 83 * and attaches the new to the Conduit. 84 */ 85 virtual MediaConduitErrorCode AttachRenderer(RefPtr<VideoRenderer> aVideoRenderer) override; 86 virtual void DetachRenderer() override; 87 88 /** 89 * APIs used by the registered external transport to this Conduit to 90 * feed in received RTP Frames to the VideoEngine for decoding 91 */ 92 virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len) override; 93 94 /** 95 * APIs used by the registered external transport to this Conduit to 96 * feed in received RTP Frames to the VideoEngine for decoding 97 */ 98 virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) override; 99 100 virtual MediaConduitErrorCode StopTransmitting() override; 101 virtual MediaConduitErrorCode StartTransmitting() override; 102 virtual MediaConduitErrorCode StopReceiving() override; 103 virtual MediaConduitErrorCode StartReceiving() override; 104 105 /** 106 * Function to configure sending codec mode for different content 107 */ 108 virtual MediaConduitErrorCode ConfigureCodecMode(webrtc::VideoCodecMode) override; 109 110 /** 111 * Function to configure send codec for the video session 112 * @param sendSessionConfig: CodecConfiguration 113 * @result: On Success, the video engine is configured with passed in codec for send 114 * On failure, video engine transmit functionality is disabled. 115 * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting 116 * transmission sub-system on the engine. 117 */ 118 virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* codecInfo) override; 119 120 /** 121 * Function to configure list of receive codecs for the video session 122 * @param sendSessionConfig: CodecConfiguration 123 * @result: On Success, the video engine is configured with passed in codec for send 124 * Also the playout is enabled. 125 * On failure, video engine transmit functionality is disabled. 126 * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting 127 * transmission sub-system on the engine. 128 */ 129 virtual MediaConduitErrorCode ConfigureRecvMediaCodecs( 130 const std::vector<VideoCodecConfig* >& codecConfigList) override; 131 132 /** 133 * Register Transport for this Conduit. RTP and RTCP frames from the VideoEngine 134 * shall be passed to the registered transport for transporting externally. 135 */ 136 virtual MediaConduitErrorCode SetTransmitterTransport(RefPtr<TransportInterface> aTransport) override; 137 138 virtual MediaConduitErrorCode SetReceiverTransport(RefPtr<TransportInterface> aTransport) override; 139 140 /** 141 * Function to set the encoding bitrate limits based on incoming frame size and rate 142 * @param width, height: dimensions of the frame 143 * @param cap: user-enforced max bitrate, or 0 144 * @param aLastFramerateTenths: holds the current input framerate 145 * @param out_start, out_min, out_max: bitrate results 146 */ 147 void SelectBitrates(unsigned short width, 148 unsigned short height, 149 unsigned int cap, 150 mozilla::Atomic<int32_t, mozilla::Relaxed>& aLastFramerateTenths, 151 unsigned int& out_min, 152 unsigned int& out_start, 153 unsigned int& out_max); 154 155 /** 156 * Function to select and change the encoding resolution based on incoming frame size 157 * and current available bandwidth. 158 * @param width, height: dimensions of the frame 159 * @param frame: optional frame to submit for encoding after reconfig 160 */ 161 bool SelectSendResolution(unsigned short width, 162 unsigned short height, 163 webrtc::I420VideoFrame *frame); 164 165 /** 166 * Function to reconfigure the current send codec for a different 167 * width/height/framerate/etc. 168 * @param width, height: dimensions of the frame 169 * @param frame: optional frame to submit for encoding after reconfig 170 */ 171 nsresult ReconfigureSendCodec(unsigned short width, 172 unsigned short height, 173 webrtc::I420VideoFrame *frame); 174 175 /** 176 * Function to select and change the encoding frame rate based on incoming frame rate 177 * and max-mbps setting. 178 * @param current framerate 179 * @result new framerate 180 */ 181 unsigned int SelectSendFrameRate(unsigned int framerate) const; 182 183 /** 184 * Function to deliver a capture video frame for encoding and transport 185 * @param video_frame: pointer to captured video-frame. 186 * @param video_frame_length: size of the frame 187 * @param width, height: dimensions of the frame 188 * @param video_type: Type of the video frame - I420, RAW 189 * @param captured_time: timestamp when the frame was captured. 190 * if 0 timestamp is automatcally generated by the engine. 191 *NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked 192 * This ensures the inserted video-frames can be transmitted by the conduit 193 */ 194 virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame, 195 unsigned int video_frame_length, 196 unsigned short width, 197 unsigned short height, 198 VideoType video_type, 199 uint64_t capture_time) override; 200 virtual MediaConduitErrorCode SendVideoFrame(webrtc::I420VideoFrame& frame) override; 201 202 /** 203 * Set an external encoder object |encoder| to the payload type |pltype| 204 * for sender side codec. 205 */ 206 virtual MediaConduitErrorCode SetExternalSendCodec(VideoCodecConfig* config, 207 VideoEncoder* encoder) override; 208 209 /** 210 * Set an external decoder object |decoder| to the payload type |pltype| 211 * for receiver side codec. 212 */ 213 virtual MediaConduitErrorCode SetExternalRecvCodec(VideoCodecConfig* config, 214 VideoDecoder* decoder) override; 215 216 /** 217 * Enables use of Rtp Stream Id, and sets the extension ID. 218 */ 219 virtual MediaConduitErrorCode EnableRTPStreamIdExtension(bool enabled, uint8_t id) override; 220 221 /** 222 * Webrtc transport implementation to send and receive RTP packet. 223 * VideoConduit registers itself as ExternalTransport to the VideoEngine 224 */ 225 virtual int SendPacket(int channel, const void *data, size_t len) override; 226 227 /** 228 * Webrtc transport implementation to send and receive RTCP packet. 229 * VideoConduit registers itself as ExternalTransport to the VideoEngine 230 */ 231 virtual int SendRTCPPacket(int channel, const void *data, size_t len) override; 232 233 234 /** 235 * Webrtc External Renderer Implementation APIs. 236 * Raw I420 Frames are delivred to the VideoConduit by the VideoEngine 237 */ 238 virtual int FrameSizeChange(unsigned int, unsigned int, unsigned int) override; 239 240 virtual int DeliverFrame(unsigned char*, size_t, uint32_t , int64_t, 241 int64_t, void *handle) override; 242 243 virtual int DeliverFrame(unsigned char*, size_t, uint32_t, uint32_t, uint32_t , int64_t, 244 int64_t, void *handle); 245 246 virtual int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override; 247 248 /** 249 * Does DeliverFrame() support a null buffer and non-null handle 250 * (video texture)? 251 * B2G support it (when using HW video decoder with graphic buffer output). 252 * XXX Investigate! Especially for Android 253 */ IsTextureSupported()254 virtual bool IsTextureSupported() override { 255 #ifdef WEBRTC_GONK 256 return true; 257 #else 258 return false; 259 #endif 260 } 261 262 virtual uint64_t CodecPluginID() override; 263 SendingWidth()264 unsigned short SendingWidth() override { 265 return mSendingWidth; 266 } 267 SendingHeight()268 unsigned short SendingHeight() override { 269 return mSendingHeight; 270 } 271 SendingMaxFs()272 unsigned int SendingMaxFs() override { 273 if(mCurSendCodecConfig) { 274 return mCurSendCodecConfig->mEncodingConstraints.maxFs; 275 } 276 return 0; 277 } 278 SendingMaxFr()279 unsigned int SendingMaxFr() override { 280 if(mCurSendCodecConfig) { 281 return mCurSendCodecConfig->mEncodingConstraints.maxFps; 282 } 283 return 0; 284 } 285 286 WebrtcVideoConduit(); 287 virtual ~WebrtcVideoConduit(); 288 289 MediaConduitErrorCode InitMain(); 290 virtual MediaConduitErrorCode Init(); 291 virtual void Destroy(); 292 GetChannel()293 int GetChannel() { return mChannel; } GetVideoEngine()294 webrtc::VideoEngine* GetVideoEngine() { return mVideoEngine; } 295 bool GetLocalSSRC(unsigned int* ssrc) override; 296 bool SetLocalSSRC(unsigned int ssrc) override; 297 bool GetRemoteSSRC(unsigned int* ssrc) override; 298 bool SetLocalCNAME(const char* cname) override; 299 bool GetVideoEncoderStats(double* framerateMean, 300 double* framerateStdDev, 301 double* bitrateMean, 302 double* bitrateStdDev, 303 uint32_t* droppedFrames) override; 304 bool GetVideoDecoderStats(double* framerateMean, 305 double* framerateStdDev, 306 double* bitrateMean, 307 double* bitrateStdDev, 308 uint32_t* discardedPackets) override; 309 bool GetAVStats(int32_t* jitterBufferDelayMs, 310 int32_t* playoutBufferDelayMs, 311 int32_t* avSyncOffsetMs) override; 312 bool GetRTPStats(unsigned int* jitterMs, unsigned int* cumulativeLost) override; 313 bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp, 314 uint32_t* jitterMs, 315 uint32_t* packetsReceived, 316 uint64_t* bytesReceived, 317 uint32_t* cumulativeLost, 318 int32_t* rttMs) override; 319 bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp, 320 unsigned int* packetsSent, 321 uint64_t* bytesSent) override; 322 uint64_t MozVideoLatencyAvg(); 323 324 private: 325 DISALLOW_COPY_AND_ASSIGN(WebrtcVideoConduit); 326 OnThread(nsIEventTarget * thread)327 static inline bool OnThread(nsIEventTarget *thread) 328 { 329 bool on; 330 nsresult rv; 331 rv = thread->IsOnCurrentThread(&on); 332 333 // If the target thread has already shut down, we don't want to assert. 334 if (rv != NS_ERROR_NOT_INITIALIZED) { 335 MOZ_ASSERT(NS_SUCCEEDED(rv)); 336 } 337 338 if (NS_WARN_IF(NS_FAILED(rv))) { 339 return false; 340 } 341 return on; 342 } 343 344 //Local database of currently applied receive codecs 345 typedef std::vector<VideoCodecConfig* > RecvCodecList; 346 347 //Function to convert between WebRTC and Conduit codec structures 348 void CodecConfigToWebRTCCodec(const VideoCodecConfig* codecInfo, 349 webrtc::VideoCodec& cinst); 350 351 //Checks the codec to be applied 352 MediaConduitErrorCode ValidateCodecConfig(const VideoCodecConfig* codecInfo, bool send); 353 354 //Utility function to dump recv codec database 355 void DumpCodecDB() const; 356 357 // Video Latency Test averaging filter 358 void VideoLatencyUpdate(uint64_t new_sample); 359 360 // Utility function to determine RED and ULPFEC payload types 361 bool DetermineREDAndULPFECPayloadTypes(uint8_t &payload_type_red, uint8_t &payload_type_ulpfec); 362 363 webrtc::VideoEngine* mVideoEngine; 364 mozilla::ReentrantMonitor mTransportMonitor; 365 RefPtr<TransportInterface> mTransmitterTransport; 366 RefPtr<TransportInterface> mReceiverTransport; 367 RefPtr<VideoRenderer> mRenderer; 368 369 ScopedCustomReleasePtr<webrtc::ViEBase> mPtrViEBase; 370 ScopedCustomReleasePtr<webrtc::ViECapture> mPtrViECapture; 371 ScopedCustomReleasePtr<webrtc::ViECodec> mPtrViECodec; 372 ScopedCustomReleasePtr<webrtc::ViENetwork> mPtrViENetwork; 373 ScopedCustomReleasePtr<webrtc::ViERender> mPtrViERender; 374 ScopedCustomReleasePtr<webrtc::ViERTP_RTCP> mPtrRTP; 375 ScopedCustomReleasePtr<webrtc::ViEExternalCodec> mPtrExtCodec; 376 377 webrtc::ViEExternalCapture* mPtrExtCapture; 378 379 // Engine state we are concerned with. 380 mozilla::Atomic<bool> mEngineTransmitting; //If true ==> Transmit Sub-system is up and running 381 mozilla::Atomic<bool> mEngineReceiving; // if true ==> Receive Sus-sysmtem up and running 382 383 int mChannel; // Video Channel for this conduit 384 int mCapId; // Capturer for this conduit 385 386 Mutex mCodecMutex; // protects mCurrSendCodecConfig 387 nsAutoPtr<VideoCodecConfig> mCurSendCodecConfig; 388 bool mInReconfig; 389 390 unsigned short mLastWidth; 391 unsigned short mLastHeight; 392 unsigned short mSendingWidth; 393 unsigned short mSendingHeight; 394 unsigned short mReceivingWidth; 395 unsigned short mReceivingHeight; 396 unsigned int mSendingFramerate; 397 // scaled by *10 because Atomic<double/float> isn't supported 398 mozilla::Atomic<int32_t, mozilla::Relaxed> mLastFramerateTenths; 399 unsigned short mNumReceivingStreams; 400 bool mVideoLatencyTestEnable; 401 uint64_t mVideoLatencyAvg; 402 uint32_t mMinBitrate; 403 uint32_t mStartBitrate; 404 uint32_t mMaxBitrate; 405 uint32_t mMinBitrateEstimate; 406 407 bool mRtpStreamIdEnabled; 408 uint8_t mRtpStreamIdExtId; 409 410 static const unsigned int sAlphaNum = 7; 411 static const unsigned int sAlphaDen = 8; 412 static const unsigned int sRoundingPadding = 1024; 413 414 RefPtr<WebrtcAudioConduit> mSyncedTo; 415 416 nsAutoPtr<VideoCodecConfig> mExternalSendCodec; 417 nsAutoPtr<VideoCodecConfig> mExternalRecvCodec; 418 nsAutoPtr<VideoEncoder> mExternalSendCodecHandle; 419 nsAutoPtr<VideoDecoder> mExternalRecvCodecHandle; 420 421 // statistics object for video codec; 422 nsAutoPtr<VideoCodecStatistics> mVideoCodecStat; 423 424 nsAutoPtr<LoadManager> mLoadManager; 425 webrtc::VideoCodecMode mCodecMode; 426 }; 427 } // end namespace 428 429 #endif 430