1 /* This Source Code Form is subject to the terms of the Mozilla Public
2  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3  * You can obtain one at http://mozilla.org/MPL/2.0/. */
4 
5 #ifndef VIDEO_SESSION_H_
6 #define VIDEO_SESSION_H_
7 
8 #include "nsAutoPtr.h"
9 #include "mozilla/Attributes.h"
10 #include "mozilla/Atomics.h"
11 
12 #include "MediaConduitInterface.h"
13 #include "MediaEngineWrapper.h"
14 #include "CodecStatistics.h"
15 #include "LoadManagerFactory.h"
16 #include "LoadManager.h"
17 #include "runnable_utils.h"
18 
19 // conflicts with #include of scoped_ptr.h
20 #undef FF
21 // Video Engine Includes
22 #include "webrtc/common_types.h"
23 #ifdef FF
24 #undef FF // Avoid name collision between scoped_ptr.h and nsCRTGlue.h.
25 #endif
26 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
27 #include "webrtc/video_engine/include/vie_base.h"
28 #include "webrtc/video_engine/include/vie_capture.h"
29 #include "webrtc/video_engine/include/vie_codec.h"
30 #include "webrtc/video_engine/include/vie_external_codec.h"
31 #include "webrtc/video_engine/include/vie_render.h"
32 #include "webrtc/video_engine/include/vie_network.h"
33 #include "webrtc/video_engine/include/vie_rtp_rtcp.h"
34 
35 /** This file hosts several structures identifying different aspects
36  * of a RTP Session.
37  */
38 
39  using  webrtc::ViEBase;
40  using  webrtc::ViENetwork;
41  using  webrtc::ViECodec;
42  using  webrtc::ViECapture;
43  using  webrtc::ViERender;
44  using  webrtc::ViEExternalCapture;
45  using  webrtc::ViEExternalCodec;
46 
47 namespace mozilla {
48 
49 class WebrtcAudioConduit;
50 class nsThread;
51 
52 // Interface of external video encoder for WebRTC.
53 class WebrtcVideoEncoder:public VideoEncoder
54                          ,public webrtc::VideoEncoder
55 {};
56 
57 // Interface of external video decoder for WebRTC.
58 class WebrtcVideoDecoder:public VideoDecoder
59                          ,public webrtc::VideoDecoder
60 {};
61 
62 /**
63  * Concrete class for Video session. Hooks up
64  *  - media-source and target to external transport
65  */
66 class WebrtcVideoConduit : public VideoSessionConduit
67                          , public webrtc::Transport
68                          , public webrtc::ExternalRenderer
69 {
70 public:
71   //VoiceEngine defined constant for Payload Name Size.
72   static const unsigned int CODEC_PLNAME_SIZE;
73 
74   /**
75    * Set up A/V sync between this (incoming) VideoConduit and an audio conduit.
76    */
77   void SyncTo(WebrtcAudioConduit *aConduit);
78 
79   /**
80    * Function to attach Renderer end-point for the Media-Video conduit.
81    * @param aRenderer : Reference to the concrete Video renderer implementation
82    * Note: Multiple invocations of this API shall remove an existing renderer
83    * and attaches the new to the Conduit.
84    */
85   virtual MediaConduitErrorCode AttachRenderer(RefPtr<VideoRenderer> aVideoRenderer) override;
86   virtual void DetachRenderer() override;
87 
88   /**
89    * APIs used by the registered external transport to this Conduit to
90    * feed in received RTP Frames to the VideoEngine for decoding
91    */
92   virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len) override;
93 
94   /**
95    * APIs used by the registered external transport to this Conduit to
96    * feed in received RTP Frames to the VideoEngine for decoding
97    */
98   virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) override;
99 
100   virtual MediaConduitErrorCode StopTransmitting() override;
101   virtual MediaConduitErrorCode StartTransmitting() override;
102   virtual MediaConduitErrorCode StopReceiving() override;
103   virtual MediaConduitErrorCode StartReceiving() override;
104 
105   /**
106    * Function to configure sending codec mode for different content
107    */
108   virtual MediaConduitErrorCode ConfigureCodecMode(webrtc::VideoCodecMode) override;
109 
110    /**
111    * Function to configure send codec for the video session
112    * @param sendSessionConfig: CodecConfiguration
113    * @result: On Success, the video engine is configured with passed in codec for send
114    *          On failure, video engine transmit functionality is disabled.
115    * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
116    *        transmission sub-system on the engine.
117    */
118   virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* codecInfo) override;
119 
120   /**
121    * Function to configure list of receive codecs for the video session
122    * @param sendSessionConfig: CodecConfiguration
123    * @result: On Success, the video engine is configured with passed in codec for send
124    *          Also the playout is enabled.
125    *          On failure, video engine transmit functionality is disabled.
126    * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
127    *        transmission sub-system on the engine.
128    */
129    virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
130                                const std::vector<VideoCodecConfig* >& codecConfigList) override;
131 
132   /**
133    * Register Transport for this Conduit. RTP and RTCP frames from the VideoEngine
134    * shall be passed to the registered transport for transporting externally.
135    */
136   virtual MediaConduitErrorCode SetTransmitterTransport(RefPtr<TransportInterface> aTransport) override;
137 
138   virtual MediaConduitErrorCode SetReceiverTransport(RefPtr<TransportInterface> aTransport) override;
139 
140   /**
141    * Function to set the encoding bitrate limits based on incoming frame size and rate
142    * @param width, height: dimensions of the frame
143    * @param cap: user-enforced max bitrate, or 0
144    * @param aLastFramerateTenths: holds the current input framerate
145    * @param out_start, out_min, out_max: bitrate results
146    */
147   void SelectBitrates(unsigned short width,
148                       unsigned short height,
149                       unsigned int cap,
150                       mozilla::Atomic<int32_t, mozilla::Relaxed>& aLastFramerateTenths,
151                       unsigned int& out_min,
152                       unsigned int& out_start,
153                       unsigned int& out_max);
154 
155   /**
156    * Function to select and change the encoding resolution based on incoming frame size
157    * and current available bandwidth.
158    * @param width, height: dimensions of the frame
159    * @param frame: optional frame to submit for encoding after reconfig
160    */
161   bool SelectSendResolution(unsigned short width,
162                             unsigned short height,
163                             webrtc::I420VideoFrame *frame);
164 
165   /**
166    * Function to reconfigure the current send codec for a different
167    * width/height/framerate/etc.
168    * @param width, height: dimensions of the frame
169    * @param frame: optional frame to submit for encoding after reconfig
170    */
171   nsresult ReconfigureSendCodec(unsigned short width,
172                                 unsigned short height,
173                                 webrtc::I420VideoFrame *frame);
174 
175   /**
176    * Function to select and change the encoding frame rate based on incoming frame rate
177    * and max-mbps setting.
178    * @param current framerate
179    * @result new framerate
180    */
181   unsigned int SelectSendFrameRate(unsigned int framerate) const;
182 
183   /**
184    * Function to deliver a capture video frame for encoding and transport
185    * @param video_frame: pointer to captured video-frame.
186    * @param video_frame_length: size of the frame
187    * @param width, height: dimensions of the frame
188    * @param video_type: Type of the video frame - I420, RAW
189    * @param captured_time: timestamp when the frame was captured.
190    *                       if 0 timestamp is automatcally generated by the engine.
191    *NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked
192    *       This ensures the inserted video-frames can be transmitted by the conduit
193    */
194   virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame,
195                                                 unsigned int video_frame_length,
196                                                 unsigned short width,
197                                                 unsigned short height,
198                                                 VideoType video_type,
199                                                 uint64_t capture_time) override;
200   virtual MediaConduitErrorCode SendVideoFrame(webrtc::I420VideoFrame& frame) override;
201 
202   /**
203    * Set an external encoder object |encoder| to the payload type |pltype|
204    * for sender side codec.
205    */
206   virtual MediaConduitErrorCode SetExternalSendCodec(VideoCodecConfig* config,
207                                                      VideoEncoder* encoder) override;
208 
209   /**
210    * Set an external decoder object |decoder| to the payload type |pltype|
211    * for receiver side codec.
212    */
213   virtual MediaConduitErrorCode SetExternalRecvCodec(VideoCodecConfig* config,
214                                                      VideoDecoder* decoder) override;
215 
216   /**
217   * Enables use of Rtp Stream Id, and sets the extension ID.
218   */
219   virtual MediaConduitErrorCode EnableRTPStreamIdExtension(bool enabled, uint8_t id) override;
220 
221   /**
222    * Webrtc transport implementation to send and receive RTP packet.
223    * VideoConduit registers itself as ExternalTransport to the VideoEngine
224    */
225   virtual int SendPacket(int channel, const void *data, size_t len) override;
226 
227   /**
228    * Webrtc transport implementation to send and receive RTCP packet.
229    * VideoConduit registers itself as ExternalTransport to the VideoEngine
230    */
231   virtual int SendRTCPPacket(int channel, const void *data, size_t len) override;
232 
233 
234   /**
235    * Webrtc External Renderer Implementation APIs.
236    * Raw I420 Frames are delivred to the VideoConduit by the VideoEngine
237    */
238   virtual int FrameSizeChange(unsigned int, unsigned int, unsigned int) override;
239 
240   virtual int DeliverFrame(unsigned char*, size_t, uint32_t , int64_t,
241                            int64_t, void *handle) override;
242 
243   virtual int DeliverFrame(unsigned char*, size_t, uint32_t, uint32_t, uint32_t , int64_t,
244                            int64_t, void *handle);
245 
246   virtual int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override;
247 
248   /**
249    * Does DeliverFrame() support a null buffer and non-null handle
250    * (video texture)?
251    * B2G support it (when using HW video decoder with graphic buffer output).
252    * XXX Investigate!  Especially for Android
253    */
IsTextureSupported()254   virtual bool IsTextureSupported() override {
255 #ifdef WEBRTC_GONK
256     return true;
257 #else
258     return false;
259 #endif
260   }
261 
262   virtual uint64_t CodecPluginID() override;
263 
SendingWidth()264   unsigned short SendingWidth() override {
265     return mSendingWidth;
266   }
267 
SendingHeight()268   unsigned short SendingHeight() override {
269     return mSendingHeight;
270   }
271 
SendingMaxFs()272   unsigned int SendingMaxFs() override {
273     if(mCurSendCodecConfig) {
274       return mCurSendCodecConfig->mEncodingConstraints.maxFs;
275     }
276     return 0;
277   }
278 
SendingMaxFr()279   unsigned int SendingMaxFr() override {
280     if(mCurSendCodecConfig) {
281       return mCurSendCodecConfig->mEncodingConstraints.maxFps;
282     }
283     return 0;
284   }
285 
286   WebrtcVideoConduit();
287   virtual ~WebrtcVideoConduit();
288 
289   MediaConduitErrorCode InitMain();
290   virtual MediaConduitErrorCode Init();
291   virtual void Destroy();
292 
GetChannel()293   int GetChannel() { return mChannel; }
GetVideoEngine()294   webrtc::VideoEngine* GetVideoEngine() { return mVideoEngine; }
295   bool GetLocalSSRC(unsigned int* ssrc) override;
296   bool SetLocalSSRC(unsigned int ssrc) override;
297   bool GetRemoteSSRC(unsigned int* ssrc) override;
298   bool SetLocalCNAME(const char* cname) override;
299   bool GetVideoEncoderStats(double* framerateMean,
300                             double* framerateStdDev,
301                             double* bitrateMean,
302                             double* bitrateStdDev,
303                             uint32_t* droppedFrames) override;
304   bool GetVideoDecoderStats(double* framerateMean,
305                             double* framerateStdDev,
306                             double* bitrateMean,
307                             double* bitrateStdDev,
308                             uint32_t* discardedPackets) override;
309   bool GetAVStats(int32_t* jitterBufferDelayMs,
310                   int32_t* playoutBufferDelayMs,
311                   int32_t* avSyncOffsetMs) override;
312   bool GetRTPStats(unsigned int* jitterMs, unsigned int* cumulativeLost) override;
313   bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp,
314                              uint32_t* jitterMs,
315                              uint32_t* packetsReceived,
316                              uint64_t* bytesReceived,
317                              uint32_t* cumulativeLost,
318                              int32_t* rttMs) override;
319   bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp,
320                            unsigned int* packetsSent,
321                            uint64_t* bytesSent) override;
322   uint64_t MozVideoLatencyAvg();
323 
324 private:
325   DISALLOW_COPY_AND_ASSIGN(WebrtcVideoConduit);
326 
OnThread(nsIEventTarget * thread)327   static inline bool OnThread(nsIEventTarget *thread)
328   {
329     bool on;
330     nsresult rv;
331     rv = thread->IsOnCurrentThread(&on);
332 
333     // If the target thread has already shut down, we don't want to assert.
334     if (rv != NS_ERROR_NOT_INITIALIZED) {
335       MOZ_ASSERT(NS_SUCCEEDED(rv));
336     }
337 
338     if (NS_WARN_IF(NS_FAILED(rv))) {
339       return false;
340     }
341     return on;
342   }
343 
344   //Local database of currently applied receive codecs
345   typedef std::vector<VideoCodecConfig* > RecvCodecList;
346 
347   //Function to convert between WebRTC and Conduit codec structures
348   void CodecConfigToWebRTCCodec(const VideoCodecConfig* codecInfo,
349                                 webrtc::VideoCodec& cinst);
350 
351   //Checks the codec to be applied
352   MediaConduitErrorCode ValidateCodecConfig(const VideoCodecConfig* codecInfo, bool send);
353 
354   //Utility function to dump recv codec database
355   void DumpCodecDB() const;
356 
357   // Video Latency Test averaging filter
358   void VideoLatencyUpdate(uint64_t new_sample);
359 
360   // Utility function to determine RED and ULPFEC payload types
361   bool DetermineREDAndULPFECPayloadTypes(uint8_t &payload_type_red, uint8_t &payload_type_ulpfec);
362 
363   webrtc::VideoEngine* mVideoEngine;
364   mozilla::ReentrantMonitor mTransportMonitor;
365   RefPtr<TransportInterface> mTransmitterTransport;
366   RefPtr<TransportInterface> mReceiverTransport;
367   RefPtr<VideoRenderer> mRenderer;
368 
369   ScopedCustomReleasePtr<webrtc::ViEBase> mPtrViEBase;
370   ScopedCustomReleasePtr<webrtc::ViECapture> mPtrViECapture;
371   ScopedCustomReleasePtr<webrtc::ViECodec> mPtrViECodec;
372   ScopedCustomReleasePtr<webrtc::ViENetwork> mPtrViENetwork;
373   ScopedCustomReleasePtr<webrtc::ViERender> mPtrViERender;
374   ScopedCustomReleasePtr<webrtc::ViERTP_RTCP> mPtrRTP;
375   ScopedCustomReleasePtr<webrtc::ViEExternalCodec> mPtrExtCodec;
376 
377   webrtc::ViEExternalCapture* mPtrExtCapture;
378 
379   // Engine state we are concerned with.
380   mozilla::Atomic<bool> mEngineTransmitting; //If true ==> Transmit Sub-system is up and running
381   mozilla::Atomic<bool> mEngineReceiving;    // if true ==> Receive Sus-sysmtem up and running
382 
383   int mChannel; // Video Channel for this conduit
384   int mCapId;   // Capturer for this conduit
385 
386   Mutex mCodecMutex; // protects mCurrSendCodecConfig
387   nsAutoPtr<VideoCodecConfig> mCurSendCodecConfig;
388   bool mInReconfig;
389 
390   unsigned short mLastWidth;
391   unsigned short mLastHeight;
392   unsigned short mSendingWidth;
393   unsigned short mSendingHeight;
394   unsigned short mReceivingWidth;
395   unsigned short mReceivingHeight;
396   unsigned int   mSendingFramerate;
397   // scaled by *10 because Atomic<double/float> isn't supported
398   mozilla::Atomic<int32_t, mozilla::Relaxed> mLastFramerateTenths;
399   unsigned short mNumReceivingStreams;
400   bool mVideoLatencyTestEnable;
401   uint64_t mVideoLatencyAvg;
402   uint32_t mMinBitrate;
403   uint32_t mStartBitrate;
404   uint32_t mMaxBitrate;
405   uint32_t mMinBitrateEstimate;
406 
407   bool mRtpStreamIdEnabled;
408   uint8_t mRtpStreamIdExtId;
409 
410   static const unsigned int sAlphaNum = 7;
411   static const unsigned int sAlphaDen = 8;
412   static const unsigned int sRoundingPadding = 1024;
413 
414   RefPtr<WebrtcAudioConduit> mSyncedTo;
415 
416   nsAutoPtr<VideoCodecConfig> mExternalSendCodec;
417   nsAutoPtr<VideoCodecConfig> mExternalRecvCodec;
418   nsAutoPtr<VideoEncoder> mExternalSendCodecHandle;
419   nsAutoPtr<VideoDecoder> mExternalRecvCodecHandle;
420 
421   // statistics object for video codec;
422   nsAutoPtr<VideoCodecStatistics> mVideoCodecStat;
423 
424   nsAutoPtr<LoadManager> mLoadManager;
425   webrtc::VideoCodecMode mCodecMode;
426 };
427 } // end namespace
428 
429 #endif
430