1 /*
2  * libjingle
3  * Copyright 2004--2011, Google Inc.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  *  1. Redistributions of source code must retain the above copyright notice,
9  *     this list of conditions and the following disclaimer.
10  *  2. Redistributions in binary form must reproduce the above copyright notice,
11  *     this list of conditions and the following disclaimer in the documentation
12  *     and/or other materials provided with the distribution.
13  *  3. The name of the author may not be used to endorse or promote products
14  *     derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 #ifdef HAVE_WEBRTC_VIDEO
33 
34 #include "talk/session/phone/webrtcvideoengine.h"
35 
36 #include "talk/base/basictypes.h"
37 #include "talk/base/common.h"
38 #include "talk/base/buffer.h"
39 #include "talk/base/byteorder.h"
40 #include "talk/base/logging.h"
41 #include "talk/base/stringutils.h"
42 #include "talk/session/phone/filevideocapturer.h"
43 #include "talk/session/phone/rtputils.h"
44 #include "talk/session/phone/streamparams.h"
45 #include "talk/session/phone/videorenderer.h"
46 #include "talk/session/phone/webrtcpassthroughrender.h"
47 #include "talk/session/phone/webrtcvoiceengine.h"
48 #include "talk/session/phone/webrtcvideocapturer.h"
49 #include "talk/session/phone/webrtcvideoframe.h"
50 #include "talk/session/phone/webrtcvie.h"
51 #include "talk/session/phone/webrtcvoe.h"
52 
53 namespace cricket {
54 
55 static const int kDefaultLogSeverity = talk_base::LS_WARNING;
56 
57 static const int kMinVideoBitrate = 100;
58 static const int kStartVideoBitrate = 300;
59 static const int kMaxVideoBitrate = 2000;
60 static const int kDefaultConferenceModeMaxVideoBitrate = 500;
61 
62 static const int kVideoMtu = 1200;
63 
64 static const int kVideoRtpBufferSize = 65536;
65 
66 static const char kVp8PayloadName[] = "VP8";
67 static const char kRedPayloadName[] = "red";
68 static const char kFecPayloadName[] = "ulpfec";
69 
70 static const int kDefaultNumberOfTemporalLayers = 3;
71 
LogMultiline(talk_base::LoggingSeverity sev,char * text)72 static void LogMultiline(talk_base::LoggingSeverity sev, char* text) {
73   const char* delim = "\r\n";
74   for (char* tok = strtok(text, delim); tok; tok = strtok(NULL, delim)) {
75     LOG_V(sev) << tok;
76   }
77 }
78 
79 class WebRtcRenderAdapter : public webrtc::ExternalRenderer {
80  public:
WebRtcRenderAdapter(VideoRenderer * renderer)81   explicit WebRtcRenderAdapter(VideoRenderer* renderer)
82       : renderer_(renderer), width_(0), height_(0) {
83   }
~WebRtcRenderAdapter()84   virtual ~WebRtcRenderAdapter() {
85   }
86 
SetRenderer(VideoRenderer * renderer)87   void SetRenderer(VideoRenderer* renderer) {
88     talk_base::CritScope cs(&crit_);
89     renderer_ = renderer;
90     // FrameSizeChange may have already been called when renderer was not set.
91     // If so we should call SetSize here.
92     // TODO: Add unit test for this case. Didn't do it now
93     // because the WebRtcRenderAdapter is currently hiding in cc file. No
94     // good way to get access to it from the unit test.
95     if (width_ > 0 && height_ > 0 && renderer_ != NULL) {
96       if (!renderer_->SetSize(width_, height_, 0)) {
97         LOG(LS_ERROR)
98             << "WebRtcRenderAdapter SetRenderer failed to SetSize to: "
99             << width_ << "x" << height_;
100       }
101     }
102   }
103   // Implementation of webrtc::ExternalRenderer.
FrameSizeChange(unsigned int width,unsigned int height,unsigned int)104   virtual int FrameSizeChange(unsigned int width, unsigned int height,
105                               unsigned int /*number_of_streams*/) {
106     talk_base::CritScope cs(&crit_);
107     width_ = width;
108     height_ = height;
109     LOG(LS_INFO) << "WebRtcRenderAdapter frame size changed to: "
110                  << width << "x" << height;
111     if (renderer_ == NULL) {
112       LOG(LS_VERBOSE) << "WebRtcRenderAdapter the renderer has not been set. "
113                       << "SetSize will be called later in SetRenderer.";
114       return 0;
115     }
116     return renderer_->SetSize(width_, height_, 0) ? 0 : -1;
117   }
DeliverFrame(unsigned char * buffer,int buffer_size,uint32_t time_stamp,int64_t render_time)118   virtual int DeliverFrame(unsigned char* buffer, int buffer_size,
119                            uint32_t time_stamp, int64_t render_time) {
120     talk_base::CritScope cs(&crit_);
121     frame_rate_tracker_.Update(1);
122     if (renderer_ == NULL) {
123       return 0;
124     }
125     WebRtcVideoFrame video_frame;
126     // Convert 90K rtp timestamp to ns timestamp.
127     int64 rtp_time_stamp_in_ns = (time_stamp / 90) *
128         talk_base::kNumNanosecsPerMillisec;
129     // Convert milisecond render time to ns timestamp.
130     int64 render_time_stamp_in_ns = render_time *
131         talk_base::kNumNanosecsPerMillisec;
132     // Send the rtp timestamp to renderer as the VideoFrame timestamp.
133     // and the render timestamp as the VideoFrame elapsed_time.
134     video_frame.Attach(buffer, buffer_size, width_, height_,
135                        1, 1, render_time_stamp_in_ns,
136                        rtp_time_stamp_in_ns, 0);
137 
138     // Sanity check on decoded frame size.
139     if (buffer_size != static_cast<int>(VideoFrame::SizeOf(width_, height_))) {
140       LOG(LS_WARNING) << "WebRtcRenderAdapter received a strange frame size: "
141                       << buffer_size;
142     }
143 
144     int ret = renderer_->RenderFrame(&video_frame) ? 0 : -1;
145     uint8* buffer_temp;
146     size_t buffer_size_temp;
147     video_frame.Detach(&buffer_temp, &buffer_size_temp);
148     return ret;
149   }
150 
width()151   unsigned int width() {
152     talk_base::CritScope cs(&crit_);
153     return width_;
154   }
height()155   unsigned int height() {
156     talk_base::CritScope cs(&crit_);
157     return height_;
158   }
framerate()159   int framerate() {
160     talk_base::CritScope cs(&crit_);
161     return frame_rate_tracker_.units_second();
162   }
renderer()163   VideoRenderer* renderer() {
164     talk_base::CritScope cs(&crit_);
165     return renderer_;
166   }
167 
168  private:
169   talk_base::CriticalSection crit_;
170   VideoRenderer* renderer_;
171   unsigned int width_;
172   unsigned int height_;
173   talk_base::RateTracker frame_rate_tracker_;
174 };
175 
176 class WebRtcDecoderObserver : public webrtc::ViEDecoderObserver {
177  public:
WebRtcDecoderObserver(int video_channel)178   explicit WebRtcDecoderObserver(int video_channel)
179        : video_channel_(video_channel),
180          framerate_(0),
181          bitrate_(0),
182          firs_requested_(0) {
183   }
184 
185   // virtual functions from VieDecoderObserver.
IncomingCodecChanged(const int videoChannel,const webrtc::VideoCodec & videoCodec)186   virtual void IncomingCodecChanged(const int videoChannel,
187                                     const webrtc::VideoCodec& videoCodec) {}
IncomingRate(const int videoChannel,const unsigned int framerate,const unsigned int bitrate)188   virtual void IncomingRate(const int videoChannel,
189                             const unsigned int framerate,
190                             const unsigned int bitrate) {
191     ASSERT(video_channel_ == videoChannel);
192     framerate_ = framerate;
193     bitrate_ = bitrate;
194   }
RequestNewKeyFrame(const int videoChannel)195   virtual void RequestNewKeyFrame(const int videoChannel) {
196     ASSERT(video_channel_ == videoChannel);
197     ++firs_requested_;
198   }
199 
framerate() const200   int framerate() const { return framerate_; }
bitrate() const201   int bitrate() const { return bitrate_; }
firs_requested() const202   int firs_requested() const { return firs_requested_; }
203 
204  private:
205   int video_channel_;
206   int framerate_;
207   int bitrate_;
208   int firs_requested_;
209 };
210 
211 class WebRtcEncoderObserver : public webrtc::ViEEncoderObserver {
212  public:
WebRtcEncoderObserver(int video_channel)213   explicit WebRtcEncoderObserver(int video_channel)
214       : video_channel_(video_channel),
215         framerate_(0),
216         bitrate_(0) {
217   }
218 
219   // virtual functions from VieEncoderObserver.
OutgoingRate(const int videoChannel,const unsigned int framerate,const unsigned int bitrate)220   virtual void OutgoingRate(const int videoChannel,
221                             const unsigned int framerate,
222                             const unsigned int bitrate) {
223     ASSERT(video_channel_ == videoChannel);
224     framerate_ = framerate;
225     bitrate_ = bitrate;
226   }
227 
framerate() const228   int framerate() const { return framerate_; }
bitrate() const229   int bitrate() const { return bitrate_; }
230 
231  private:
232   int video_channel_;
233   int framerate_;
234   int bitrate_;
235 };
236 
237 class WebRtcLocalStreamInfo {
238  public:
width()239   int width() {
240     talk_base::CritScope cs(&crit_);
241     return width_;
242   }
height()243   int height() {
244     talk_base::CritScope cs(&crit_);
245     return height_;
246   }
framerate()247   int framerate() {
248     talk_base::CritScope cs(&crit_);
249     return rate_tracker_.units_second();
250   }
251 
UpdateFrame(int width,int height)252   void UpdateFrame(int width, int height) {
253     talk_base::CritScope cs(&crit_);
254     width_ = width;
255     height_ = height;
256     rate_tracker_.Update(1);
257   }
258 
259  private:
260   talk_base::CriticalSection crit_;
261   unsigned int width_;
262   unsigned int height_;
263   talk_base::RateTracker rate_tracker_;
264 };
265 
266 // WebRtcVideoChannelInfo is a container class with members such as renderer
267 // and a decoder observer that is used by receive channels.
268 // It must exist as long as the receive channel is connected to renderer or a
269 // decoder observer in this class and methods in the class should only be called
270 // from the worker thread.
271 class WebRtcVideoChannelInfo  {
272  public:
WebRtcVideoChannelInfo(int channel_id)273   explicit WebRtcVideoChannelInfo(int channel_id)
274       : channel_id_(channel_id),
275         render_adapter_(NULL),
276         decoder_observer_(channel_id) {
277   }
channel_id()278   int channel_id() { return channel_id_; }
SetRenderer(VideoRenderer * renderer)279   void SetRenderer(VideoRenderer* renderer) {
280     render_adapter_.SetRenderer(renderer);
281   }
render_adapter()282   WebRtcRenderAdapter* render_adapter() { return &render_adapter_; }
decoder_observer()283   WebRtcDecoderObserver* decoder_observer() { return &decoder_observer_; }
284 
285  private:
286   int channel_id_;  // Webrtc video channel number.
287   // Renderer for this channel.
288   WebRtcRenderAdapter render_adapter_;
289   WebRtcDecoderObserver decoder_observer_;
290 };
291 
292 const WebRtcVideoEngine::VideoCodecPref
293     WebRtcVideoEngine::kVideoCodecPrefs[] = {
294     {kVp8PayloadName, 100, 0},
295     {kRedPayloadName, 101, 1},
296     {kFecPayloadName, 102, 2},
297 };
298 
299 static const int64 kNsPerFrame = 33333333;  // 30fps
300 
301 // The formats are sorted by the descending order of width. We use the order to
302 // find the next format for CPU and bandwidth adaptation.
303 const VideoFormatPod WebRtcVideoEngine::kVideoFormats[] = {
304   {1280, 800, kNsPerFrame, FOURCC_ANY},
305   {1280, 720, kNsPerFrame, FOURCC_ANY},
306   {960, 600, kNsPerFrame, FOURCC_ANY},
307   {960, 540, kNsPerFrame, FOURCC_ANY},
308   {640, 400, kNsPerFrame, FOURCC_ANY},
309   {640, 360, kNsPerFrame, FOURCC_ANY},
310   {640, 480, kNsPerFrame, FOURCC_ANY},
311   {480, 300, kNsPerFrame, FOURCC_ANY},
312   {480, 270, kNsPerFrame, FOURCC_ANY},
313   {480, 360, kNsPerFrame, FOURCC_ANY},
314   {320, 200, kNsPerFrame, FOURCC_ANY},
315   {320, 180, kNsPerFrame, FOURCC_ANY},
316   {320, 240, kNsPerFrame, FOURCC_ANY},
317   {240, 150, kNsPerFrame, FOURCC_ANY},
318   {240, 135, kNsPerFrame, FOURCC_ANY},
319   {240, 180, kNsPerFrame, FOURCC_ANY},
320   {160, 100, kNsPerFrame, FOURCC_ANY},
321   {160, 90, kNsPerFrame, FOURCC_ANY},
322   {160, 120, kNsPerFrame, FOURCC_ANY},
323 };
324 
325 const VideoFormatPod WebRtcVideoEngine::kDefaultVideoFormat =
326     {640, 400, kNsPerFrame, FOURCC_ANY};
327 
WebRtcVideoEngine()328 WebRtcVideoEngine::WebRtcVideoEngine() {
329   Construct(new ViEWrapper(), new ViETraceWrapper(), NULL);
330 }
331 
WebRtcVideoEngine(WebRtcVoiceEngine * voice_engine,ViEWrapper * vie_wrapper)332 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
333                                      ViEWrapper* vie_wrapper) {
334   Construct(vie_wrapper, new ViETraceWrapper(), voice_engine);
335 }
336 
WebRtcVideoEngine(WebRtcVoiceEngine * voice_engine,ViEWrapper * vie_wrapper,ViETraceWrapper * tracing)337 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
338                                      ViEWrapper* vie_wrapper,
339                                      ViETraceWrapper* tracing) {
340   Construct(vie_wrapper, tracing, voice_engine);
341 }
342 
Construct(ViEWrapper * vie_wrapper,ViETraceWrapper * tracing,WebRtcVoiceEngine * voice_engine)343 void WebRtcVideoEngine::Construct(ViEWrapper* vie_wrapper,
344                                   ViETraceWrapper* tracing,
345                                   WebRtcVoiceEngine* voice_engine) {
346   LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine";
347   vie_wrapper_.reset(vie_wrapper);
348   vie_wrapper_base_initialized_ = false;
349   tracing_.reset(tracing);
350   voice_engine_ = voice_engine;
351   initialized_ = false;
352   log_level_ = kDefaultLogSeverity;
353   render_module_.reset(new WebRtcPassthroughRender());
354   local_renderer_w_ = local_renderer_h_ = 0;
355   local_renderer_ = NULL;
356   owns_capturer_ = false;
357   video_capturer_ = NULL;
358   capture_started_ = false;
359 
360   ApplyLogging("");
361   if (tracing_->SetTraceCallback(this) != 0) {
362     LOG_RTCERR1(SetTraceCallback, this);
363   }
364 
365   // Set default quality levels for our supported codecs. We override them here
366   // if we know your cpu performance is low, and they can be updated explicitly
367   // by calling SetDefaultCodec.  For example by a flute preference setting, or
368   // by the server with a jec in response to our reported system info.
369   VideoCodec max_codec(kVideoCodecPrefs[0].payload_type,
370                        kVideoCodecPrefs[0].name,
371                        kDefaultVideoFormat.width,
372                        kDefaultVideoFormat.height,
373                        VideoFormat::IntervalToFps(kDefaultVideoFormat.interval),
374                        0);
375   if (!SetDefaultCodec(max_codec)) {
376     LOG(LS_ERROR) << "Failed to initialize list of supported codec types";
377   }
378 }
379 
~WebRtcVideoEngine()380 WebRtcVideoEngine::~WebRtcVideoEngine() {
381   ClearCapturer();
382   LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
383   if (initialized_) {
384     Terminate();
385   }
386   tracing_->SetTraceCallback(NULL);
387   // Test to see if the media processor was deregistered properly.
388   ASSERT(SignalMediaFrame.is_empty());
389 }
390 
Init()391 bool WebRtcVideoEngine::Init() {
392   LOG(LS_INFO) << "WebRtcVideoEngine::Init";
393   bool result = InitVideoEngine();
394   if (result) {
395     LOG(LS_INFO) << "VideoEngine Init done";
396   } else {
397     LOG(LS_ERROR) << "VideoEngine Init failed, releasing";
398     Terminate();
399   }
400   return result;
401 }
402 
InitVideoEngine()403 bool WebRtcVideoEngine::InitVideoEngine() {
404   LOG(LS_INFO) << "WebRtcVideoEngine::InitVideoEngine";
405 
406   // Init WebRTC VideoEngine.
407   if (!vie_wrapper_base_initialized_) {
408     if (vie_wrapper_->base()->Init() != 0) {
409       LOG_RTCERR0(Init);
410       return false;
411     }
412     vie_wrapper_base_initialized_ = true;
413   }
414 
415   // Log the VoiceEngine version info.
416   char buffer[1024] = "";
417   if (vie_wrapper_->base()->GetVersion(buffer) != 0) {
418     LOG_RTCERR0(GetVersion);
419     return false;
420   }
421 
422   LOG(LS_INFO) << "WebRtc VideoEngine Version:";
423   LogMultiline(talk_base::LS_INFO, buffer);
424 
425   // Hook up to VoiceEngine for sync purposes, if supplied.
426   if (!voice_engine_) {
427     LOG(LS_WARNING) << "NULL voice engine";
428   } else if ((vie_wrapper_->base()->SetVoiceEngine(
429       voice_engine_->voe()->engine())) != 0) {
430     LOG_RTCERR0(SetVoiceEngine);
431     return false;
432   }
433 
434   // Register for callbacks from the engine.
435   if ((vie_wrapper_->base()->RegisterObserver(*this)) != 0) {
436     LOG_RTCERR0(RegisterObserver);
437     return false;
438   }
439 
440   // Register our custom render module.
441   if (vie_wrapper_->render()->RegisterVideoRenderModule(
442       *render_module_.get()) != 0) {
443     LOG_RTCERR0(RegisterVideoRenderModule);
444     return false;
445   }
446 
447   initialized_ = true;
448   return true;
449 }
450 
Terminate()451 void WebRtcVideoEngine::Terminate() {
452   LOG(LS_INFO) << "WebRtcVideoEngine::Terminate";
453   initialized_ = false;
454   SetCapture(false);
455 
456   if (vie_wrapper_->render()->DeRegisterVideoRenderModule(
457       *render_module_.get()) != 0) {
458     LOG_RTCERR0(DeRegisterVideoRenderModule);
459   }
460 
461   if (vie_wrapper_->base()->DeregisterObserver() != 0) {
462     LOG_RTCERR0(DeregisterObserver);
463   }
464 
465   if (vie_wrapper_->base()->SetVoiceEngine(NULL) != 0) {
466     LOG_RTCERR0(SetVoiceEngine);
467   }
468 }
469 
GetCapabilities()470 int WebRtcVideoEngine::GetCapabilities() {
471   return VIDEO_RECV | VIDEO_SEND;
472 }
473 
SetOptions(int options)474 bool WebRtcVideoEngine::SetOptions(int options) {
475   return true;
476 }
477 
SetDefaultEncoderConfig(const VideoEncoderConfig & config)478 bool WebRtcVideoEngine::SetDefaultEncoderConfig(
479     const VideoEncoderConfig& config) {
480   return SetDefaultCodec(config.max_codec);
481 }
482 
483 // SetDefaultCodec may be called while the capturer is running. For example, a
484 // test call is started in a page with QVGA default codec, and then a real call
485 // is started in another page with VGA default codec. This is the corner case
486 // and happens only when a session is started. We ignore this case currently.
SetDefaultCodec(const VideoCodec & codec)487 bool WebRtcVideoEngine::SetDefaultCodec(const VideoCodec& codec) {
488   if (!RebuildCodecList(codec)) {
489     LOG(LS_WARNING) << "Failed to RebuildCodecList";
490     return false;
491   }
492 
493   default_codec_format_ = VideoFormat(
494       video_codecs_[0].width,
495       video_codecs_[0].height,
496       VideoFormat::FpsToInterval(video_codecs_[0].framerate),
497       FOURCC_ANY);
498   return true;
499 }
500 
CreateChannel(VoiceMediaChannel * voice_channel)501 WebRtcVideoMediaChannel* WebRtcVideoEngine::CreateChannel(
502     VoiceMediaChannel* voice_channel) {
503   WebRtcVideoMediaChannel* channel =
504       new WebRtcVideoMediaChannel(this, voice_channel);
505   if (!channel->Init()) {
506     delete channel;
507     channel = NULL;
508   }
509   return channel;
510 }
511 
SetCaptureDevice(const Device * device)512 bool WebRtcVideoEngine::SetCaptureDevice(const Device* device) {
513   const bool owns_capturer = true;
514   if (!device) {
515     if (!SetCapturer(NULL, owns_capturer)) {
516       return false;
517     }
518     LOG(LS_INFO) << "Camera set to NULL";
519     return true;
520   }
521   // No-op if the device hasn't changed.
522   if ((video_capturer_ != NULL) && video_capturer_->GetId() == device->id) {
523     return true;
524   }
525   // Create a new capturer for the specified device.
526   VideoCapturer* capturer = CreateVideoCapturer(*device);
527   if (!capturer) {
528     LOG(LS_ERROR) << "Failed to create camera '" << device->name << "', id='"
529                   << device->id << "'";
530     return false;
531   }
532   if (!SetCapturer(capturer, owns_capturer)) {
533     return false;
534   }
535   LOG(LS_INFO) << "Camera set to '" << device->name << "', id='"
536                << device->id << "'";
537   return true;
538 }
539 
SetCaptureModule(webrtc::VideoCaptureModule * vcm)540 bool WebRtcVideoEngine::SetCaptureModule(webrtc::VideoCaptureModule* vcm) {
541   const bool owns_capturer = true;
542   if (!vcm) {
543     if (!SetCapturer(NULL, owns_capturer)) {
544       return false;
545     }
546     LOG(LS_INFO) << "Camera set to NULL";
547     return true;
548   }
549   // Create a new capturer for the specified device.
550   WebRtcVideoCapturer* capturer = new WebRtcVideoCapturer;
551   if (!capturer->Init(vcm)) {
552     LOG(LS_ERROR) << "Failed to create camera from VCM";
553     delete capturer;
554     return false;
555   }
556   if (!SetCapturer(capturer, owns_capturer)) {
557     return false;
558   }
559   LOG(LS_INFO) << "Camera created with VCM";
560   CaptureResult ret = SetCapture(true);
561   if (ret != cricket::CR_SUCCESS && ret != cricket::CR_PENDING) {
562     LOG(LS_ERROR) << "Failed to start camera.";
563     return false;
564   }
565   return true;
566 }
567 
SetVideoCapturer(VideoCapturer * capturer,uint32)568 bool WebRtcVideoEngine::SetVideoCapturer(VideoCapturer* capturer,
569                                          uint32 /*ssrc*/) {
570   const bool owns_capturer = false;
571   return SetCapturer(capturer, owns_capturer);
572 }
573 
SetLocalRenderer(VideoRenderer * renderer)574 bool WebRtcVideoEngine::SetLocalRenderer(VideoRenderer* renderer) {
575   local_renderer_w_ = local_renderer_h_ = 0;
576   local_renderer_ = renderer;
577   return true;
578 }
579 
SetCapture(bool capture)580 CaptureResult WebRtcVideoEngine::SetCapture(bool capture) {
581   bool old_capture = capture_started_;
582   capture_started_ = capture;
583   CaptureResult res = UpdateCapturingState();
584   if (res != CR_SUCCESS && res != CR_PENDING) {
585     capture_started_ = old_capture;
586   }
587   return res;
588 }
589 
CreateVideoCapturer(const Device & device)590 VideoCapturer* WebRtcVideoEngine::CreateVideoCapturer(const Device& device) {
591   if (FileVideoCapturer::IsFileVideoCapturerDevice(device)) {
592     FileVideoCapturer* capturer = new FileVideoCapturer;
593     if (!capturer->Init(device)) {
594       delete capturer;
595       return NULL;
596     }
597     capturer->set_repeat(talk_base::kForever);
598     return capturer;
599   } else {
600     WebRtcVideoCapturer* capturer = new WebRtcVideoCapturer;
601     if (!capturer->Init(device)) {
602       delete capturer;
603       return NULL;
604     }
605     return capturer;
606   }
607 }
608 
UpdateCapturingState()609 CaptureResult WebRtcVideoEngine::UpdateCapturingState() {
610   CaptureResult result = CR_SUCCESS;
611 
612   bool capture = capture_started_;
613   if (!IsCapturing() && capture) {  // Start capturing.
614     if (video_capturer_ == NULL) {
615       return CR_NO_DEVICE;
616     }
617 
618     VideoFormat capture_format;
619     if (!video_capturer_->GetBestCaptureFormat(default_codec_format_,
620                                                &capture_format)) {
621       LOG(LS_WARNING) << "Unsupported format:"
622                       << " width=" << default_codec_format_.width
623                       << " height=" << default_codec_format_.height
624                       << ". Supported formats are:";
625       const std::vector<VideoFormat>* formats =
626           video_capturer_->GetSupportedFormats();
627       if (formats) {
628         for (std::vector<VideoFormat>::const_iterator i = formats->begin();
629              i != formats->end(); ++i) {
630           const VideoFormat& format = *i;
631           LOG(LS_WARNING) << "  " << GetFourccName(format.fourcc) << ":"
632                           << format.width << "x" << format.height << "x"
633                           << format.framerate();
634         }
635       }
636       return CR_FAILURE;
637     }
638 
639     // Start the video capturer.
640     result = video_capturer_->Start(capture_format);
641     if (CR_SUCCESS != result && CR_PENDING != result) {
642       LOG(LS_ERROR) << "Failed to start the video capturer";
643       return result;
644     }
645   } else if (IsCapturing() && !capture) {  // Stop capturing.
646     video_capturer_->Stop();
647   }
648 
649   return result;
650 }
651 
IsCapturing() const652 bool WebRtcVideoEngine::IsCapturing() const {
653   return (video_capturer_ != NULL) && video_capturer_->IsRunning();
654 }
655 
OnFrameCaptured(VideoCapturer * capturer,const CapturedFrame * frame)656 void WebRtcVideoEngine::OnFrameCaptured(VideoCapturer* capturer,
657                                         const CapturedFrame* frame) {
658   // Force 16:10 for now. We'll be smarter with the capture refactor.
659   int cropped_height = frame->width * default_codec_format_.height
660       / default_codec_format_.width;
661   if (cropped_height > frame->height) {
662     // TODO: Once we support horizontal cropping, add cropped_width.
663     cropped_height = frame->height;
664   }
665 
666   // This CapturedFrame* will already be in I420. In the future, when
667   // WebRtcVideoFrame has support for independent planes, we can just attach
668   // to it and update the pointers when cropping.
669   WebRtcVideoFrame i420_frame;
670   if (!i420_frame.Init(frame, frame->width, cropped_height)) {
671     LOG(LS_ERROR) << "Couldn't convert to I420! "
672                   << frame->width << " x " << cropped_height;
673     return;
674   }
675 
676   // TODO: This is the trigger point for Tx video processing.
677   // Once the capturer refactoring is done, we will move this into the
678   // capturer...it's not there right now because that image is in not in the
679   // I420 color space.
680   // The clients that subscribe will obtain meta info from the frame.
681   // When this trigger is switched over to capturer, need to pass in the real
682   // ssrc.
683   {
684     talk_base::CritScope cs(&signal_media_critical_);
685     SignalMediaFrame(kDummyVideoSsrc, &i420_frame);
686   }
687 
688   // Send I420 frame to the local renderer.
689   if (local_renderer_) {
690     if (local_renderer_w_ != static_cast<int>(i420_frame.GetWidth()) ||
691         local_renderer_h_ != static_cast<int>(i420_frame.GetHeight())) {
692       local_renderer_->SetSize(local_renderer_w_ = i420_frame.GetWidth(),
693                                local_renderer_h_ = i420_frame.GetHeight(), 0);
694     }
695     local_renderer_->RenderFrame(&i420_frame);
696   }
697 
698   // Send I420 frame to the registered senders.
699   talk_base::CritScope cs(&channels_crit_);
700   for (VideoChannels::iterator it = channels_.begin();
701       it != channels_.end(); ++it) {
702     if ((*it)->sending()) (*it)->SendFrame(0, &i420_frame);
703   }
704 }
705 
codecs() const706 const std::vector<VideoCodec>& WebRtcVideoEngine::codecs() const {
707   return video_codecs_;
708 }
709 
SetLogging(int min_sev,const char * filter)710 void WebRtcVideoEngine::SetLogging(int min_sev, const char* filter) {
711   // if min_sev == -1, we keep the current log level.
712   if (min_sev >= 0) {
713     log_level_ = min_sev;
714   }
715   ApplyLogging(filter);
716 }
717 
GetLastEngineError()718 int WebRtcVideoEngine::GetLastEngineError() {
719   return vie_wrapper_->error();
720 }
721 
722 // Checks to see whether we comprehend and could receive a particular codec
FindCodec(const VideoCodec & in)723 bool WebRtcVideoEngine::FindCodec(const VideoCodec& in) {
724   for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) {
725     const VideoFormat fmt(kVideoFormats[i]);
726     if ((in.width == 0 && in.height == 0) ||
727         (fmt.width == in.width && fmt.height == in.height)) {
728       for (int j = 0; j < ARRAY_SIZE(kVideoCodecPrefs); ++j) {
729         VideoCodec codec(kVideoCodecPrefs[j].payload_type,
730                          kVideoCodecPrefs[j].name, 0, 0, 0, 0);
731         if (codec.Matches(in)) {
732           return true;
733         }
734       }
735     }
736   }
737   return false;
738 }
739 
740 // Given the requested codec, returns true if we can send that codec type and
741 // updates out with the best quality we could send for that codec. If current is
742 // not empty, we constrain out so that its aspect ratio matches current's.
CanSendCodec(const VideoCodec & requested,const VideoCodec & current,VideoCodec * out)743 bool WebRtcVideoEngine::CanSendCodec(const VideoCodec& requested,
744                                      const VideoCodec& current,
745                                      VideoCodec* out) {
746   if (!out) {
747     return false;
748   }
749 
750   std::vector<VideoCodec>::const_iterator local_max;
751   for (local_max = video_codecs_.begin();
752        local_max < video_codecs_.end();
753        ++local_max) {
754     // First match codecs by payload type
755     if (!requested.Matches(local_max->id, local_max->name)) {
756       continue;
757     }
758 
759     out->id = requested.id;
760     out->name = requested.name;
761     out->preference = requested.preference;
762     out->framerate = talk_base::_min(requested.framerate, local_max->framerate);
763     out->width = 0;
764     out->height = 0;
765 
766     if (0 == requested.width && 0 == requested.height) {
767       // Special case with resolution 0. The channel should not send frames.
768       return true;
769     } else if (0 == requested.width || 0 == requested.height) {
770       // 0xn and nx0 are invalid resolutions.
771       return false;
772     }
773 
774     // Pick the best quality that is within their and our bounds and has the
775     // correct aspect ratio.
776     for (int j = 0; j < ARRAY_SIZE(kVideoFormats); ++j) {
777       const VideoFormat format(kVideoFormats[j]);
778 
779       // Skip any format that is larger than the local or remote maximums, or
780       // smaller than the current best match
781       if (format.width > requested.width || format.height > requested.height ||
782           format.width > local_max->width ||
783           (format.width < out->width && format.height < out->height)) {
784         continue;
785       }
786 
787       bool better = false;
788 
789       // Check any further constraints on this prospective format
790       if (!out->width || !out->height) {
791         // If we don't have any matches yet, this is the best so far.
792         better = true;
793       } else if (current.width && current.height) {
794         // current is set so format must match its ratio exactly.
795         better =
796             (format.width * current.height == format.height * current.width);
797       } else {
798         // Prefer closer aspect ratios i.e
799         // format.aspect - requested.aspect < out.aspect - requested.aspect
800         better = abs(format.width * requested.height * out->height -
801                      requested.width * format.height * out->height) <
802                  abs(out->width * format.height * requested.height -
803                      requested.width * format.height * out->height);
804       }
805 
806       if (better) {
807         out->width = format.width;
808         out->height = format.height;
809       }
810     }
811     if (out->width > 0) {
812       return true;
813     }
814   }
815   return false;
816 }
817 
ConvertToCricketVideoCodec(const webrtc::VideoCodec & in_codec,VideoCodec * out_codec)818 void WebRtcVideoEngine::ConvertToCricketVideoCodec(
819     const webrtc::VideoCodec& in_codec, VideoCodec* out_codec) {
820   out_codec->id = in_codec.plType;
821   out_codec->name = in_codec.plName;
822   out_codec->width = in_codec.width;
823   out_codec->height = in_codec.height;
824   out_codec->framerate = in_codec.maxFramerate;
825 }
826 
ConvertFromCricketVideoCodec(const VideoCodec & in_codec,webrtc::VideoCodec * out_codec)827 bool WebRtcVideoEngine::ConvertFromCricketVideoCodec(
828     const VideoCodec& in_codec, webrtc::VideoCodec* out_codec) {
829   bool found = false;
830   int ncodecs = vie_wrapper_->codec()->NumberOfCodecs();
831   for (int i = 0; i < ncodecs; ++i) {
832     if (vie_wrapper_->codec()->GetCodec(i, *out_codec) == 0 &&
833         _stricmp(in_codec.name.c_str(), out_codec->plName) == 0) {
834       found = true;
835       break;
836     }
837   }
838 
839   if (!found) {
840     LOG(LS_ERROR) << "invalid codec type";
841     return false;
842   }
843 
844   if (in_codec.id != 0)
845     out_codec->plType = in_codec.id;
846 
847   if (in_codec.width != 0)
848     out_codec->width = in_codec.width;
849 
850   if (in_codec.height != 0)
851     out_codec->height = in_codec.height;
852 
853   if (in_codec.framerate != 0)
854     out_codec->maxFramerate = in_codec.framerate;
855 
856   // Init the codec with the default bandwidth options.
857   out_codec->minBitrate = kMinVideoBitrate;
858   out_codec->startBitrate = kStartVideoBitrate;
859   out_codec->maxBitrate = kMaxVideoBitrate;
860 
861   return true;
862 }
863 
RegisterChannel(WebRtcVideoMediaChannel * channel)864 void WebRtcVideoEngine::RegisterChannel(WebRtcVideoMediaChannel *channel) {
865   talk_base::CritScope cs(&channels_crit_);
866   channels_.push_back(channel);
867 }
868 
UnregisterChannel(WebRtcVideoMediaChannel * channel)869 void WebRtcVideoEngine::UnregisterChannel(WebRtcVideoMediaChannel *channel) {
870   talk_base::CritScope cs(&channels_crit_);
871   channels_.erase(std::remove(channels_.begin(), channels_.end(), channel),
872                   channels_.end());
873 }
874 
SetVoiceEngine(WebRtcVoiceEngine * voice_engine)875 bool WebRtcVideoEngine::SetVoiceEngine(WebRtcVoiceEngine* voice_engine) {
876   if (initialized_) {
877     LOG(LS_WARNING) << "SetVoiceEngine can not be called after Init";
878     return false;
879   }
880   voice_engine_ = voice_engine;
881   return true;
882 }
883 
EnableTimedRender()884 bool WebRtcVideoEngine::EnableTimedRender() {
885   if (initialized_) {
886     LOG(LS_WARNING) << "EnableTimedRender can not be called after Init";
887     return false;
888   }
889   render_module_.reset(webrtc::VideoRender::CreateVideoRender(0, NULL,
890       false, webrtc::kRenderExternal));
891   return true;
892 }
893 
894 // See https://sites.google.com/a/google.com/wavelet/
895 //     Home/Magic-Flute--RTC-Engine-/Magic-Flute-Command-Line-Parameters
896 // for all supported command line setttings.
ApplyLogging(const std::string & log_filter)897 void WebRtcVideoEngine::ApplyLogging(const std::string& log_filter) {
898   int filter = 0;
899   switch (log_level_) {
900     case talk_base::LS_VERBOSE: filter |= webrtc::kTraceAll;
901     case talk_base::LS_INFO: filter |= webrtc::kTraceStateInfo;
902     case talk_base::LS_WARNING: filter |= webrtc::kTraceWarning;
903     case talk_base::LS_ERROR: filter |=
904         webrtc::kTraceError | webrtc::kTraceCritical;
905   }
906   tracing_->SetTraceFilter(filter);
907 
908   // Set WebRTC trace file.
909   std::vector<std::string> opts;
910   talk_base::tokenize(log_filter, ' ', '"', '"', &opts);
911   std::vector<std::string>::iterator tracefile =
912       std::find(opts.begin(), opts.end(), "tracefile");
913   if (tracefile != opts.end() && ++tracefile != opts.end()) {
914     // Write WebRTC debug output (at same loglevel) to file
915     if (tracing_->SetTraceFile(tracefile->c_str()) == -1) {
916       LOG_RTCERR1(SetTraceFile, *tracefile);
917     }
918   }
919 }
920 
921 // Rebuilds the codec list to be only those that are less intensive
922 // than the specified codec.
RebuildCodecList(const VideoCodec & in_codec)923 bool WebRtcVideoEngine::RebuildCodecList(const VideoCodec& in_codec) {
924   if (!FindCodec(in_codec))
925     return false;
926 
927   video_codecs_.clear();
928 
929   bool found = false;
930   for (size_t i = 0; i < ARRAY_SIZE(kVideoCodecPrefs); ++i) {
931     const VideoCodecPref& pref(kVideoCodecPrefs[i]);
932     if (!found)
933       found = (in_codec.name == pref.name);
934     if (found) {
935       VideoCodec codec(pref.payload_type, pref.name,
936                        in_codec.width, in_codec.height, in_codec.framerate,
937                        ARRAY_SIZE(kVideoCodecPrefs) - i);
938       video_codecs_.push_back(codec);
939     }
940   }
941   ASSERT(found);
942   return true;
943 }
944 
SetCapturer(VideoCapturer * capturer,bool own_capturer)945 bool WebRtcVideoEngine::SetCapturer(VideoCapturer* capturer,
946                                     bool own_capturer) {
947   if (capturer == NULL) {
948     // Stop capturing before clearing the capturer.
949     if (SetCapture(false) != CR_SUCCESS) {
950       LOG(LS_WARNING) << "Camera failed to stop";
951       return false;
952     }
953     ClearCapturer();
954     return true;
955   }
956   // Hook up signals and install the supplied capturer.
957   SignalCaptureResult.repeat(capturer->SignalStartResult);
958   capturer->SignalFrameCaptured.connect(this,
959       &WebRtcVideoEngine::OnFrameCaptured);
960   ClearCapturer();
961   video_capturer_ = capturer;
962   owns_capturer_ = own_capturer;
963   // Possibly restart the capturer if it is supposed to be running.
964   CaptureResult result = UpdateCapturingState();
965   if (result != CR_SUCCESS && result != CR_PENDING) {
966     LOG(LS_WARNING) << "Camera failed to restart";
967     return false;
968   }
969   return true;
970 }
971 
PerformanceAlarm(const unsigned int cpu_load)972 void WebRtcVideoEngine::PerformanceAlarm(const unsigned int cpu_load) {
973   LOG(LS_INFO) << "WebRtcVideoEngine::PerformanceAlarm";
974 }
975 
976 // Ignore spammy trace messages, mostly from the stats API when we haven't
977 // gotten RTCP info yet from the remote side.
ShouldIgnoreTrace(const std::string & trace)978 bool WebRtcVideoEngine::ShouldIgnoreTrace(const std::string& trace) {
979   static const char* const kTracesToIgnore[] = {
980     NULL
981   };
982   for (const char* const* p = kTracesToIgnore; *p; ++p) {
983     if (trace.find(*p) == 0) {
984       return true;
985     }
986   }
987   return false;
988 }
989 
GetNumOfChannels()990 int WebRtcVideoEngine::GetNumOfChannels() {
991   talk_base::CritScope cs(&channels_crit_);
992   return channels_.size();
993 }
994 
Print(const webrtc::TraceLevel level,const char * trace,const int length)995 void WebRtcVideoEngine::Print(const webrtc::TraceLevel level,
996                               const char* trace, const int length) {
997   talk_base::LoggingSeverity sev = talk_base::LS_VERBOSE;
998   if (level == webrtc::kTraceError || level == webrtc::kTraceCritical)
999     sev = talk_base::LS_ERROR;
1000   else if (level == webrtc::kTraceWarning)
1001     sev = talk_base::LS_WARNING;
1002   else if (level == webrtc::kTraceStateInfo || level == webrtc::kTraceInfo)
1003     sev = talk_base::LS_INFO;
1004 
1005   if (sev >= log_level_) {
1006     // Skip past boilerplate prefix text
1007     if (length < 72) {
1008       std::string msg(trace, length);
1009       LOG(LS_ERROR) << "Malformed webrtc log message: ";
1010       LOG_V(sev) << msg;
1011     } else {
1012       std::string msg(trace + 71, length - 72);
1013       if (!ShouldIgnoreTrace(msg) &&
1014           (!voice_engine_ || !voice_engine_->ShouldIgnoreTrace(msg))) {
1015         LOG_V(sev) << "WebRtc:" << msg;
1016       }
1017     }
1018   }
1019 }
1020 
RegisterProcessor(VideoProcessor * video_processor)1021 bool WebRtcVideoEngine::RegisterProcessor(
1022     VideoProcessor* video_processor) {
1023   talk_base::CritScope cs(&signal_media_critical_);
1024   SignalMediaFrame.connect(video_processor,
1025                            &VideoProcessor::OnFrame);
1026   return true;
1027 }
UnregisterProcessor(VideoProcessor * video_processor)1028 bool WebRtcVideoEngine::UnregisterProcessor(
1029     VideoProcessor* video_processor) {
1030   talk_base::CritScope cs(&signal_media_critical_);
1031   SignalMediaFrame.disconnect(video_processor);
1032   return true;
1033 }
1034 
ClearCapturer()1035 void WebRtcVideoEngine::ClearCapturer() {
1036   if (owns_capturer_) {
1037     delete video_capturer_;
1038   }
1039   video_capturer_ = NULL;
1040 }
1041 
1042 // WebRtcVideoMediaChannel
1043 
WebRtcVideoMediaChannel(WebRtcVideoEngine * engine,VoiceMediaChannel * channel)1044 WebRtcVideoMediaChannel::WebRtcVideoMediaChannel(
1045     WebRtcVideoEngine* engine, VoiceMediaChannel* channel)
1046     : engine_(engine),
1047       voice_channel_(channel),
1048       vie_channel_(-1),
1049       vie_capture_(-1),
1050       external_capture_(NULL),
1051       render_started_(false),
1052       muted_(false),
1053       first_receive_ssrc_(0),
1054       send_min_bitrate_(kMinVideoBitrate),
1055       send_start_bitrate_(kStartVideoBitrate),
1056       send_max_bitrate_(kMaxVideoBitrate),
1057       sending_(false),
1058       local_stream_info_(new WebRtcLocalStreamInfo()),
1059       options_(0) {
1060   engine->RegisterChannel(this);
1061 }
1062 
Init()1063 bool WebRtcVideoMediaChannel::Init() {
1064   if (engine_->vie()->base()->CreateChannel(vie_channel_) != 0) {
1065     LOG_RTCERR1(CreateChannel, vie_channel_);
1066     return false;
1067   }
1068   if (!ConfigureChannel(vie_channel_)) {
1069     engine_->vie()->base()->DeleteChannel(vie_channel_);
1070     vie_channel_ = -1;
1071     return false;
1072   }
1073 
1074   if (!ConfigureReceiving(vie_channel_, 0)) {
1075     engine_->vie()->base()->DeleteChannel(vie_channel_);
1076     vie_channel_ = -1;
1077     return false;
1078   }
1079 
1080   LOG(LS_INFO) << "WebRtcVideoMediaChannel::Init "
1081                << "vie_channel " << vie_channel_ << " created";
1082 
1083   // Register external capture.
1084   if (engine()->vie()->capture()->AllocateExternalCaptureDevice(
1085       vie_capture_, external_capture_) != 0) {
1086     LOG_RTCERR0(AllocateExternalCaptureDevice);
1087     return false;
1088   }
1089 
1090   // Connect external capture.
1091   if (engine()->vie()->capture()->ConnectCaptureDevice(
1092       vie_capture_, vie_channel_) != 0) {
1093     LOG_RTCERR2(ConnectCaptureDevice, vie_capture_, vie_channel_);
1094     return false;
1095   }
1096 
1097   // Register encoder observer for outgoing framerate and bitrate.
1098   encoder_observer_.reset(new WebRtcEncoderObserver(vie_channel_));
1099   if (engine()->vie()->codec()->RegisterEncoderObserver(
1100       vie_channel_, *encoder_observer_) != 0) {
1101     LOG_RTCERR1(RegisterEncoderObserver, encoder_observer_.get());
1102     return false;
1103   }
1104 
1105   return true;
1106 }
1107 
~WebRtcVideoMediaChannel()1108 WebRtcVideoMediaChannel::~WebRtcVideoMediaChannel() {
1109   if (vie_channel_ != -1) {
1110     // Stop sending.
1111     SetSend(false);
1112     if (engine()->vie()->codec()->DeregisterEncoderObserver(
1113         vie_channel_) != 0) {
1114       LOG_RTCERR1(DeregisterEncoderObserver, vie_channel_);
1115     }
1116 
1117     // Stop the renderer.
1118     SetRender(false);
1119 
1120     // Destroy the external capture interface.
1121     if (vie_capture_ != -1) {
1122       if (engine()->vie()->capture()->DisconnectCaptureDevice(
1123           vie_channel_) != 0) {
1124         LOG_RTCERR1(DisconnectCaptureDevice, vie_channel_);
1125       }
1126       if (engine()->vie()->capture()->ReleaseCaptureDevice(
1127           vie_capture_) != 0) {
1128         LOG_RTCERR1(ReleaseCaptureDevice, vie_capture_);
1129       }
1130     }
1131 
1132     // Remove all receive streams and the default channel.
1133     while (!mux_channels_.empty()) {
1134       RemoveRecvStream(mux_channels_.begin()->first);
1135     }
1136   }
1137 
1138   // Unregister the channel from the engine.
1139   engine()->UnregisterChannel(this);
1140 }
1141 
SetRecvCodecs(const std::vector<VideoCodec> & codecs)1142 bool WebRtcVideoMediaChannel::SetRecvCodecs(
1143     const std::vector<VideoCodec>& codecs) {
1144   receive_codecs_.clear();
1145   for (std::vector<VideoCodec>::const_iterator iter = codecs.begin();
1146       iter != codecs.end(); ++iter) {
1147     if (engine()->FindCodec(*iter)) {
1148       webrtc::VideoCodec wcodec;
1149       if (engine()->ConvertFromCricketVideoCodec(*iter, &wcodec)) {
1150         receive_codecs_.push_back(wcodec);
1151       }
1152     } else {
1153       LOG(LS_INFO) << "Unknown codec " << iter->name;
1154       return false;
1155     }
1156   }
1157 
1158   for (ChannelMap::iterator it = mux_channels_.begin();
1159       it != mux_channels_.end(); ++it) {
1160     if (!SetReceiveCodecs(it->second->channel_id()))
1161       return false;
1162   }
1163   return true;
1164 }
1165 
SetSendCodecs(const std::vector<VideoCodec> & codecs)1166 bool WebRtcVideoMediaChannel::SetSendCodecs(
1167     const std::vector<VideoCodec>& codecs) {
1168   // Match with local video codec list.
1169   std::vector<webrtc::VideoCodec> send_codecs;
1170   int red_type = -1, fec_type = -1;
1171   VideoCodec checked_codec;
1172   VideoCodec current;  // defaults to 0x0
1173   if (sending_) {
1174     engine()->ConvertToCricketVideoCodec(*send_codec_, &current);
1175   }
1176   for (std::vector<VideoCodec>::const_iterator iter = codecs.begin();
1177       iter != codecs.end(); ++iter) {
1178     if (_stricmp(iter->name.c_str(), kRedPayloadName) == 0) {
1179       red_type = iter->id;
1180     } else if (_stricmp(iter->name.c_str(), kFecPayloadName) == 0) {
1181       fec_type = iter->id;
1182     } else if (engine()->CanSendCodec(*iter, current, &checked_codec)) {
1183       webrtc::VideoCodec wcodec;
1184       if (engine()->ConvertFromCricketVideoCodec(checked_codec, &wcodec)) {
1185         send_codecs.push_back(wcodec);
1186       }
1187     } else {
1188       LOG(LS_WARNING) << "Unknown codec " << iter->name;
1189     }
1190   }
1191 
1192   // Fail if we don't have a match.
1193   if (send_codecs.empty()) {
1194     LOG(LS_WARNING) << "No matching codecs avilable";
1195     return false;
1196   }
1197 
1198   // Configure video protection.
1199   if (!SetNackFec(vie_channel_, red_type, fec_type)) {
1200     return false;
1201   }
1202 
1203   // Select the first matched codec.
1204   webrtc::VideoCodec& codec(send_codecs[0]);
1205 
1206   // Set the default number of temporal layers for VP8.
1207   if (webrtc::kVideoCodecVP8 == codec.codecType) {
1208     codec.codecSpecific.VP8.numberOfTemporalLayers =
1209         kDefaultNumberOfTemporalLayers;
1210     // Turn off the VP8 error resilience
1211     codec.codecSpecific.VP8.resilience = webrtc::kResilienceOff;
1212   }
1213 
1214   if (!SetSendCodec(
1215       codec, send_min_bitrate_, send_start_bitrate_, send_max_bitrate_)) {
1216     return false;
1217   }
1218   LogSendCodecChange("SetSendCodecs()");
1219 
1220   return true;
1221 }
1222 
SetSendStreamFormat(uint32 ssrc,const VideoFormat & format)1223 bool WebRtcVideoMediaChannel::SetSendStreamFormat(uint32 ssrc,
1224                                                   const VideoFormat& format) {
1225   if (send_params_.get() == NULL) {
1226     LOG(LS_WARNING) << "Sending stream has not been added yet.";
1227     return false;
1228   }
1229   if (!send_params_->has_ssrc(ssrc)) {
1230     LOG(LS_ERROR) << "The specified ssrc " << ssrc << " is not in use.";
1231     return false;
1232   }
1233 
1234   if (send_codec_.get() == NULL) {
1235     LOG(LS_ERROR) << "The send codec has not been set yet.";
1236     return false;
1237   }
1238 
1239   webrtc::VideoCodec codec = *send_codec_.get();
1240   codec.width = format.width;
1241   codec.height = format.height;
1242   codec.maxFramerate = VideoFormat::IntervalToFps(format.interval);
1243 
1244   bool ret = SetSendCodec(
1245       codec, send_min_bitrate_, send_start_bitrate_, send_max_bitrate_);
1246   if (ret) {
1247     LogSendCodecChange("SetSendStreamFormat()");
1248   }
1249   return ret;
1250 }
1251 
SetRender(bool render)1252 bool WebRtcVideoMediaChannel::SetRender(bool render) {
1253   if (render == render_started_) {
1254     return true;  // no action required
1255   }
1256 
1257   bool ret = true;
1258   for (ChannelMap::iterator it = mux_channels_.begin();
1259       it != mux_channels_.end(); ++it) {
1260     if (render) {
1261       if (engine()->vie()->render()->StartRender(
1262           it->second->channel_id()) != 0) {
1263         LOG_RTCERR1(StartRender, it->second->channel_id());
1264         ret = false;
1265       }
1266     } else {
1267       if (engine()->vie()->render()->StopRender(
1268           it->second->channel_id()) != 0) {
1269         LOG_RTCERR1(StopRender, it->second->channel_id());
1270         ret = false;
1271       }
1272     }
1273   }
1274   if (ret) {
1275     render_started_ = render;
1276   }
1277 
1278   return ret;
1279 }
1280 
SetSend(bool send)1281 bool WebRtcVideoMediaChannel::SetSend(bool send) {
1282   if (send_params_.get() == NULL && send) {
1283     LOG(LS_ERROR) << "No stream added";
1284     return false;
1285   }
1286   if (send == sending()) {
1287     return true;  // No action required.
1288   }
1289 
1290   if (send) {
1291     // We've been asked to start sending.
1292     // SetSendCodecs must have been called already.
1293     if (!send_codec_.get()) {
1294       return false;
1295     }
1296     // Start send now.
1297     if (!StartSend()) {
1298       return false;
1299     }
1300   } else {
1301     // We've been asked to stop sending.
1302     if (!StopSend()) {
1303       return false;
1304     }
1305   }
1306   sending_ = send;
1307 
1308   return true;
1309 }
1310 
GetChannelNum(uint32 ssrc)1311 int WebRtcVideoMediaChannel::GetChannelNum(uint32 ssrc) {
1312   ChannelMap::iterator it = mux_channels_.find(ssrc);
1313   return (it != mux_channels_.end()) ? it->second->channel_id() : -1;
1314 }
1315 
AddSendStream(const StreamParams & sp)1316 bool WebRtcVideoMediaChannel::AddSendStream(const StreamParams& sp) {
1317   if (send_params_.get() != NULL) {
1318     LOG(LS_ERROR) << "WebRtcVideoMediaChannel supports one sending channel";
1319     return false;
1320   }
1321 
1322   if (!IsOneSsrcStream(sp)) {
1323       LOG(LS_ERROR) << "AddSendStream: bad local stream parameters";
1324       return false;
1325   }
1326 
1327   // Set the send (local) SSRC.
1328   // If there are multiple send SSRCs, we can only set the first one here, and
1329   // the rest of the SSRC(s) need to be set after SetSendCodec has been called
1330   // (with a codec requires multiple SSRC(s)).
1331   if (engine()->vie()->rtp()->SetLocalSSRC(vie_channel_,
1332                                            sp.first_ssrc()) != 0) {
1333     LOG_RTCERR2(SetLocalSSRC, vie_channel_, sp.first_ssrc());
1334     return false;
1335   }
1336 
1337   // Set RTCP CName.
1338   if (engine()->vie()->rtp()->SetRTCPCName(vie_channel_,
1339                                            sp.cname.c_str()) != 0) {
1340     LOG_RTCERR2(SetRTCPCName, vie_channel_, sp.cname.c_str());
1341     return false;
1342   }
1343 
1344   // Set the SSRC on the receive channels and this send channel.
1345   // Receive channels have to have the same SSRC in order to send receiver
1346   // reports with this SSRC.
1347   for (ChannelMap::const_iterator it = mux_channels_.begin();
1348        it != mux_channels_.end(); ++it) {
1349     WebRtcVideoChannelInfo* info = it->second;
1350     int channel_id = info->channel_id();
1351     if (engine()->vie()->rtp()->SetLocalSSRC(channel_id,
1352                                              sp.first_ssrc()) != 0) {
1353       LOG_RTCERR1(SetLocalSSRC, it->first);
1354       return false;
1355     }
1356   }
1357 
1358   // Save the StreamParams.
1359   send_params_.reset(new StreamParams(sp));
1360 
1361   // Reset send codec after stream parameters changed.
1362   if (send_codec_.get() != NULL) {
1363     if (!SetSendCodec(*send_codec_, send_min_bitrate_,
1364                       send_start_bitrate_, send_max_bitrate_)) {
1365       return false;
1366     }
1367     LogSendCodecChange("SetSendStreamFormat()");
1368   }
1369 
1370   if (sending_) {
1371     return StartSend();
1372   }
1373   return true;
1374 }
1375 
RemoveSendStream(uint32 ssrc)1376 bool WebRtcVideoMediaChannel::RemoveSendStream(uint32 ssrc) {
1377   if (send_params_.get() == NULL || !send_params_->has_ssrc(ssrc)) {
1378     LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc
1379                     << " which doesn't exist.";
1380     return false;
1381   }
1382   if (sending_) {
1383     StopSend();
1384   }
1385   send_params_.reset(NULL);
1386   return true;
1387 }
1388 
AddRecvStream(const StreamParams & sp)1389 bool WebRtcVideoMediaChannel::AddRecvStream(const StreamParams& sp) {
1390   // TODO Remove this once BWE works properly across different send
1391   // and receive channels.
1392   // Reuse default channel for recv stream in 1:1 call.
1393   if ((options_ & OPT_CONFERENCE) == 0 && first_receive_ssrc_ == 0) {
1394     LOG(LS_INFO) << "Recv stream " << sp.first_ssrc()
1395                  << " reuse default channel #"
1396                  << vie_channel_;
1397     first_receive_ssrc_ = sp.first_ssrc();
1398     return true;
1399   }
1400 
1401   if (mux_channels_.find(sp.first_ssrc()) != mux_channels_.end()) {
1402     LOG(LS_ERROR) << "Stream already exists";
1403     return false;
1404   }
1405 
1406   // TODO: Implement recv media from multiple SSRCs per stream.
1407   if (sp.ssrcs.size() != 1) {
1408     LOG(LS_ERROR) << "WebRtcVideoMediaChannel supports one receiving SSRC per"
1409                   << " stream";
1410     return false;
1411   }
1412 
1413   // Create a new channel for receiving video data.
1414   // In order to get the bandwidth estimation work fine for
1415   // receive only channels, we connect all receiving channels
1416   // to our master send channel.
1417   int channel_id = -1;
1418   if (engine_->vie()->base()->CreateReceiveChannel(channel_id,
1419                                                    vie_channel_) != 0) {
1420     LOG_RTCERR2(CreateReceiveChannel, channel_id, vie_channel_);
1421     return false;
1422   }
1423 
1424   // Get the default renderer.
1425   VideoRenderer* default_renderer = NULL;
1426   if ((options_ & OPT_CONFERENCE) != 0) {
1427     if (mux_channels_.size() == 1 &&
1428         mux_channels_.find(0) != mux_channels_.end()) {
1429       GetRenderer(0, &default_renderer);
1430     }
1431   }
1432 
1433   if (!ConfigureChannel(channel_id) ||
1434       !ConfigureReceiving(channel_id, sp.first_ssrc())) {
1435     engine_->vie()->base()->DeleteChannel(channel_id);
1436     return false;
1437   }
1438 
1439   // The first recv stream reuses the default renderer (if a default renderer
1440   // has been set).
1441   if (default_renderer) {
1442     SetRenderer(sp.first_ssrc(), default_renderer);
1443   }
1444 
1445   LOG(LS_INFO) << "New video stream " << sp.first_ssrc()
1446                << " registered to VideoEngine channel #"
1447                << channel_id << " and connected to channel #" << vie_channel_;
1448 
1449   return true;
1450 }
1451 
RemoveRecvStream(uint32 ssrc)1452 bool WebRtcVideoMediaChannel::RemoveRecvStream(uint32 ssrc) {
1453   ChannelMap::iterator it = mux_channels_.find(ssrc);
1454 
1455   if (it == mux_channels_.end()) {
1456     // TODO: Remove this once BWE works properly across different send
1457     // and receive channels.
1458     // The default channel is reused for recv stream in 1:1 call.
1459     if (first_receive_ssrc_ == ssrc) {
1460       first_receive_ssrc_ = 0;
1461       return true;
1462     }
1463     return false;
1464   }
1465   WebRtcVideoChannelInfo* info = it->second;
1466   int channel_id = info->channel_id();
1467   if (engine()->vie()->render()->RemoveRenderer(channel_id) != 0) {
1468     LOG_RTCERR1(RemoveRenderer, channel_id);
1469   }
1470 
1471   if (engine()->vie()->network()->DeregisterSendTransport(channel_id) !=0) {
1472     LOG_RTCERR1(DeRegisterSendTransport, channel_id);
1473   }
1474 
1475   if (engine()->vie()->codec()->DeregisterDecoderObserver(
1476       channel_id) != 0) {
1477     LOG_RTCERR1(DeregisterDecoderObserver, channel_id);
1478   }
1479 
1480   LOG(LS_INFO) << "Removing video stream " << ssrc
1481                << " with VideoEngine channel #"
1482                << channel_id;
1483   if (engine()->vie()->base()->DeleteChannel(channel_id) == -1) {
1484     LOG_RTCERR1(DeleteChannel, channel_id);
1485     // Leak the WebRtcVideoChannelInfo owned by |it| but remove the channel from
1486     // mux_channels_.
1487     mux_channels_.erase(it);
1488     return false;
1489   }
1490   // Delete the WebRtcVideoChannelInfo pointed to by it->second.
1491   delete info;
1492   mux_channels_.erase(it);
1493   return true;
1494 }
1495 
StartSend()1496 bool WebRtcVideoMediaChannel::StartSend() {
1497   if (engine()->vie()->base()->StartSend(vie_channel_) != 0) {
1498     LOG_RTCERR1(StartSend, vie_channel_);
1499     return false;
1500   }
1501 
1502   // TODO Change this once REMB supporting multiple sending channels.
1503   // Send remb (2nd param) and use remb for BWE (3rd param).
1504   if (engine_->vie()->rtp()->SetRembStatus(vie_channel_, true, true) != 0) {
1505     LOG_RTCERR3(SetRembStatus, vie_channel_, true, true);
1506     return false;
1507   }
1508 
1509   return true;
1510 }
1511 
StopSend()1512 bool WebRtcVideoMediaChannel::StopSend() {
1513   if (engine()->vie()->base()->StopSend(vie_channel_) != 0) {
1514     LOG_RTCERR1(StopSend, vie_channel_);
1515     return false;
1516   }
1517 
1518   // TODO Change this once REMB supporting multiple sending channels.
1519   // Don't send remb (2nd param) but use remb for BWE (3rd param).
1520   if (engine_->vie()->rtp()->SetRembStatus(vie_channel_, false, true) != 0) {
1521     LOG_RTCERR3(SetRembStatus, vie_channel_, false, true);
1522     return false;
1523   }
1524 
1525   return true;
1526 }
1527 
IsOneSsrcStream(const StreamParams & sp)1528 bool WebRtcVideoMediaChannel::IsOneSsrcStream(const StreamParams& sp) {
1529   return (sp.ssrcs.size() == 1 && sp.ssrc_groups.size() == 0);
1530 }
1531 
SetRenderer(uint32 ssrc,VideoRenderer * renderer)1532 bool WebRtcVideoMediaChannel::SetRenderer(uint32 ssrc,
1533                                           VideoRenderer* renderer) {
1534   if (mux_channels_.find(ssrc) == mux_channels_.end()) {
1535     // TODO: Remove this once BWE works properly across different send
1536     // and receive channels.
1537     // The default channel is reused for recv stream in 1:1 call.
1538     if (first_receive_ssrc_ == ssrc &&
1539         mux_channels_.find(0) != mux_channels_.end()) {
1540       LOG(LS_INFO) << "SetRenderer " << ssrc
1541                    << " reuse default channel #"
1542                    << vie_channel_;
1543       mux_channels_[0]->SetRenderer(renderer);
1544       return true;
1545     }
1546     return false;
1547   }
1548 
1549   mux_channels_[ssrc]->SetRenderer(renderer);
1550   return true;
1551 }
1552 
GetStats(VideoMediaInfo * info)1553 bool WebRtcVideoMediaChannel::GetStats(VideoMediaInfo* info) {
1554   // Get basic statistics.
1555   unsigned int bytes_sent, packets_sent, bytes_recv, packets_recv;
1556   unsigned int ssrc;
1557   if (engine_->vie()->rtp()->GetRTPStatistics(vie_channel_,
1558           bytes_sent, packets_sent, bytes_recv, packets_recv) != 0) {
1559     LOG_RTCERR1(GetRTPStatistics, vie_channel_);
1560     return false;
1561   }
1562 
1563   // Get sender statistics and build VideoSenderInfo.
1564   if (send_params_.get() != NULL && send_codec_.get() != NULL) {
1565     VideoSenderInfo sinfo;
1566     sinfo.ssrcs = send_params_->ssrcs;
1567     sinfo.codec_name = send_codec_.get() ? send_codec_->plName : "";
1568     sinfo.bytes_sent = bytes_sent;
1569     sinfo.packets_sent = packets_sent;
1570     sinfo.packets_cached = -1;
1571     sinfo.packets_lost = -1;
1572     sinfo.fraction_lost = -1;
1573     sinfo.firs_rcvd = -1;
1574     sinfo.nacks_rcvd = -1;
1575     sinfo.rtt_ms = -1;
1576     sinfo.frame_width = local_stream_info_->width();
1577     sinfo.frame_height = local_stream_info_->height();
1578     sinfo.framerate_input = local_stream_info_->framerate();
1579     sinfo.framerate_sent = encoder_observer_->framerate();
1580     sinfo.nominal_bitrate = encoder_observer_->bitrate();
1581     sinfo.preferred_bitrate = send_max_bitrate_;
1582 
1583     // Get received RTCP statistics for the sender, if available.
1584     // It's not a fatal error if we can't, since RTCP may not have arrived yet.
1585     uint16 r_fraction_lost;
1586     unsigned int r_cumulative_lost;
1587     unsigned int r_extended_max;
1588     unsigned int r_jitter;
1589     int r_rtt_ms;
1590     if (engine_->vie()->rtp()->GetSentRTCPStatistics(vie_channel_,
1591             r_fraction_lost, r_cumulative_lost, r_extended_max,
1592             r_jitter, r_rtt_ms) == 0) {
1593       // Convert Q8 to float.
1594       sinfo.packets_lost = r_cumulative_lost;
1595       sinfo.fraction_lost = static_cast<float>(r_fraction_lost) / (1 << 8);
1596       sinfo.rtt_ms = r_rtt_ms;
1597     }
1598     info->senders.push_back(sinfo);
1599   } else {
1600     LOG(LS_WARNING) << "GetStats: sender information not ready.";
1601   }
1602 
1603   // Get the SSRC and stats for each receiver, based on our own calculations.
1604   for (ChannelMap::const_iterator it = mux_channels_.begin();
1605        it != mux_channels_.end(); ++it) {
1606     // Don't report receive statistics from the default channel if we have
1607     // specified receive channels.
1608     if (it->first == 0 && mux_channels_.size() > 1)
1609       continue;
1610     WebRtcVideoChannelInfo* channel = it->second;
1611 
1612     // Get receiver statistics and build VideoReceiverInfo, if we have data.
1613     if (engine_->vie()->rtp()->GetRemoteSSRC(channel->channel_id(), ssrc) != 0)
1614       continue;
1615 
1616     if (engine_->vie()->rtp()->GetRTPStatistics(
1617         channel->channel_id(), bytes_sent, packets_sent, bytes_recv,
1618         packets_recv) != 0) {
1619       LOG_RTCERR1(GetRTPStatistics, channel->channel_id());
1620       return false;
1621     }
1622     VideoReceiverInfo rinfo;
1623     rinfo.ssrcs.push_back(ssrc);
1624     rinfo.bytes_rcvd = bytes_recv;
1625     rinfo.packets_rcvd = packets_recv;
1626     rinfo.packets_lost = -1;
1627     rinfo.packets_concealed = -1;
1628     rinfo.fraction_lost = -1;  // from SentRTCP
1629     rinfo.firs_sent = channel->decoder_observer()->firs_requested();
1630     rinfo.nacks_sent = -1;
1631     rinfo.frame_width = channel->render_adapter()->width();
1632     rinfo.frame_height = channel->render_adapter()->height();
1633     rinfo.framerate_rcvd = channel->decoder_observer()->framerate();
1634     int fps = channel->render_adapter()->framerate();
1635     rinfo.framerate_decoded = fps;
1636     rinfo.framerate_output = fps;
1637 
1638     // Get sent RTCP statistics.
1639     uint16 s_fraction_lost;
1640     unsigned int s_cumulative_lost;
1641     unsigned int s_extended_max;
1642     unsigned int s_jitter;
1643     int s_rtt_ms;
1644     if (engine_->vie()->rtp()->GetReceivedRTCPStatistics(channel->channel_id(),
1645             s_fraction_lost, s_cumulative_lost, s_extended_max,
1646             s_jitter, s_rtt_ms) == 0) {
1647       // Convert Q8 to float.
1648       rinfo.packets_lost = s_cumulative_lost;
1649       rinfo.fraction_lost = static_cast<float>(s_fraction_lost) / (1 << 8);
1650     }
1651     info->receivers.push_back(rinfo);
1652   }
1653 
1654   // Build BandwidthEstimationInfo.
1655   // TODO: Fill in more BWE stats once we have them.
1656   unsigned int total_bitrate_sent;
1657   unsigned int video_bitrate_sent;
1658   unsigned int fec_bitrate_sent;
1659   unsigned int nack_bitrate_sent;
1660   if (engine_->vie()->rtp()->GetBandwidthUsage(vie_channel_,
1661       total_bitrate_sent, video_bitrate_sent,
1662       fec_bitrate_sent, nack_bitrate_sent) == 0) {
1663     BandwidthEstimationInfo bwe;
1664     bwe.actual_enc_bitrate = video_bitrate_sent;
1665     bwe.transmit_bitrate = total_bitrate_sent;
1666     bwe.retransmit_bitrate = nack_bitrate_sent;
1667 
1668     // Add bandwidth estimation.
1669     unsigned int estimated_send_bandwidth;
1670     if (engine_->vie()->rtp()->GetEstimatedSendBandwidth(
1671         vie_channel_, &estimated_send_bandwidth)) {
1672       bwe.available_send_bandwidth = estimated_send_bandwidth;
1673     } else {
1674       LOG_RTCERR2(GetEstimatedSendBandwidth, vie_channel_,
1675                   &estimated_send_bandwidth);
1676     }
1677     unsigned int estimated_recv_bandwidth;
1678     if (engine_->vie()->rtp()->GetEstimatedReceiveBandwidth(
1679         vie_channel_, &estimated_recv_bandwidth)) {
1680       bwe.available_recv_bandwidth = estimated_recv_bandwidth;
1681     } else {
1682       LOG_RTCERR2(GetEstimatedRecvBandwidth, vie_channel_,
1683                   &estimated_recv_bandwidth);
1684     }
1685     unsigned int target_enc_bitrate;
1686     if (engine_->vie()->codec()->GetCodecTargetBitrate(
1687         vie_channel_, &target_enc_bitrate)) {
1688       bwe.target_enc_bitrate = target_enc_bitrate;
1689     } else {
1690       LOG_RTCERR2(GetCodecTargetBitrate, vie_channel_,
1691                   &target_enc_bitrate);
1692     }
1693 
1694     info->bw_estimations.push_back(bwe);
1695   } else {
1696     LOG_RTCERR1(GetBandwidthUsage, vie_channel_);
1697   }
1698 
1699   return true;
1700 }
1701 
SendIntraFrame()1702 bool WebRtcVideoMediaChannel::SendIntraFrame() {
1703   bool ret = true;
1704   if (engine()->vie()->codec()->SendKeyFrame(vie_channel_) != 0) {
1705     LOG_RTCERR1(SendKeyFrame, vie_channel_);
1706     ret = false;
1707   }
1708 
1709   return ret;
1710 }
1711 
RequestIntraFrame()1712 bool WebRtcVideoMediaChannel::RequestIntraFrame() {
1713   // There is no API exposed to application to request a key frame
1714   // ViE does this internally when there are errors from decoder
1715   return false;
1716 }
1717 
OnPacketReceived(talk_base::Buffer * packet)1718 void WebRtcVideoMediaChannel::OnPacketReceived(talk_base::Buffer* packet) {
1719   // Pick which channel to send this packet to. If this packet doesn't match
1720   // any multiplexed streams, just send it to the default channel. Otherwise,
1721   // send it to the specific decoder instance for that stream.
1722   uint32 ssrc = 0;
1723   if (!GetRtpSsrc(packet->data(), packet->length(), &ssrc))
1724     return;
1725   int which_channel = GetChannelNum(ssrc);
1726   if (which_channel == -1) {
1727     which_channel = video_channel();
1728   }
1729 
1730   engine()->vie()->network()->ReceivedRTPPacket(which_channel,
1731                                                 packet->data(),
1732                                                 packet->length());
1733 }
1734 
OnRtcpReceived(talk_base::Buffer * packet)1735 void WebRtcVideoMediaChannel::OnRtcpReceived(talk_base::Buffer* packet) {
1736 // Sending channels need all RTCP packets with feedback information.
1737 // Even sender reports can contain attached report blocks.
1738 // Receiving channels need sender reports in order to create
1739 // correct receiver reports.
1740 
1741   uint32 ssrc = 0;
1742   if (!GetRtcpSsrc(packet->data(), packet->length(), &ssrc)) {
1743     LOG(LS_WARNING) << "Failed to parse SSRC from received RTCP packet";
1744     return;
1745   }
1746   int type = 0;
1747   if (!GetRtcpType(packet->data(), packet->length(), & type)) {
1748     LOG(LS_WARNING) << "Failed to parse type from received RTCP packet";
1749     return;
1750   }
1751 
1752   // If it is a sender report, find the channel that is listening.
1753   if (type == kRtcpTypeSR) {
1754     int which_channel = GetChannelNum(ssrc);
1755     if (which_channel != -1 && which_channel != vie_channel_) {
1756       engine_->vie()->network()->ReceivedRTCPPacket(which_channel,
1757                                                     packet->data(),
1758                                                     packet->length());
1759     }
1760   }
1761   // The sending channel receives all RTCP packets.
1762   engine_->vie()->network()->ReceivedRTCPPacket(vie_channel_,
1763                                                 packet->data(),
1764                                                 packet->length());
1765 }
1766 
Mute(bool on)1767 bool WebRtcVideoMediaChannel::Mute(bool on) {
1768   muted_ = on;
1769   return true;
1770 }
1771 
SetSendBandwidth(bool autobw,int bps)1772 bool WebRtcVideoMediaChannel::SetSendBandwidth(bool autobw, int bps) {
1773   LOG(LS_INFO) << "WebRtcVideoMediaChanne::SetSendBandwidth";
1774 
1775   if (0 != (options_ & OPT_CONFERENCE)) {
1776     LOG(LS_INFO) << "Conference mode ignores SetSendBandWidth";
1777     return true;
1778   }
1779 
1780   if (!send_codec_.get()) {
1781     LOG(LS_INFO) << "The send codec has not been set up yet";
1782     return true;
1783   }
1784 
1785   int min_bitrate;
1786   int start_bitrate;
1787   int max_bitrate;
1788   if (autobw) {
1789     // Use the default values for min bitrate.
1790     min_bitrate = kMinVideoBitrate;
1791     // Use the default value or the bps for the max
1792     max_bitrate = (bps <= 0) ? send_max_bitrate_ : (bps / 1000);
1793     // Maximum start bitrate can be kStartVideoBitrate.
1794     start_bitrate = talk_base::_min(kStartVideoBitrate, max_bitrate);
1795   } else {
1796     // Use the default start or the bps as the target bitrate.
1797     int target_bitrate = (bps <= 0) ? kStartVideoBitrate : (bps / 1000);
1798     min_bitrate = target_bitrate;
1799     start_bitrate = target_bitrate;
1800     max_bitrate = target_bitrate;
1801   }
1802 
1803   if (!SetSendCodec(*send_codec_, min_bitrate, start_bitrate, max_bitrate)) {
1804     return false;
1805   }
1806   LogSendCodecChange("SetSendBandwidth()");
1807 
1808   return true;
1809 }
1810 
SetOptions(int options)1811 bool WebRtcVideoMediaChannel::SetOptions(int options) {
1812   // Always accept options that are unchanged.
1813   if (options_ == options) {
1814     return true;
1815   }
1816 
1817   // Reject new options if we're already sending.
1818   if (sending()) {
1819     return false;
1820   }
1821 
1822   // Save the options, to be interpreted where appropriate.
1823   options_ = options;
1824 
1825   // Adjust send codec bitrate if needed.
1826   int conf_max_bitrate = kDefaultConferenceModeMaxVideoBitrate;
1827   int expected_bitrate = (0 != (options_ & OPT_CONFERENCE)) ?
1828       conf_max_bitrate : kMaxVideoBitrate;
1829   if (NULL != send_codec_.get() && send_max_bitrate_ != expected_bitrate) {
1830     // On success, SetSendCodec() will reset send_max_bitrate_ to
1831     // expected_bitrate.
1832     if (!SetSendCodec(*send_codec_,
1833                       send_min_bitrate_,
1834                       send_start_bitrate_,
1835                       expected_bitrate)) {
1836       return false;
1837     }
1838     LogSendCodecChange("SetOptions()");
1839   }
1840 
1841   // Enable denoising if needed.
1842   if (vie_capture_ != -1) {
1843     bool enable = (options_ & OPT_VIDEO_NOISE_REDUCTION) != 0;
1844     // The EnableDenoising may return -1 when the denoising is already
1845     // enabled/disabled, which should not be treated as an error.
1846     // TODO: Return false once EnableDenoising only
1847     // reports the real failure.
1848     engine()->vie()->image()->EnableDenoising(vie_capture_, enable);
1849   } else {
1850     LOG(LS_WARNING) << "SetOptions: Video Capture is not ready.";
1851   }
1852 
1853   return true;
1854 }
1855 
SetInterface(NetworkInterface * iface)1856 void WebRtcVideoMediaChannel::SetInterface(NetworkInterface* iface) {
1857   MediaChannel::SetInterface(iface);
1858   // Set the RTP recv/send buffer to a bigger size
1859   if (network_interface_) {
1860     network_interface_->SetOption(NetworkInterface::ST_RTP,
1861                                   talk_base::Socket::OPT_RCVBUF,
1862                                   kVideoRtpBufferSize);
1863     network_interface_->SetOption(NetworkInterface::ST_RTP,
1864                                   talk_base::Socket::OPT_SNDBUF,
1865                                   kVideoRtpBufferSize);
1866   }
1867 }
1868 
GetRenderer(uint32 ssrc,VideoRenderer ** renderer)1869 bool WebRtcVideoMediaChannel::GetRenderer(uint32 ssrc,
1870                                           VideoRenderer** renderer) {
1871   ChannelMap::const_iterator it = mux_channels_.find(ssrc);
1872   if (it == mux_channels_.end()) {
1873     if (first_receive_ssrc_ == ssrc &&
1874         mux_channels_.find(0) != mux_channels_.end()) {
1875       LOG(LS_INFO) << " GetRenderer " << ssrc
1876                    << " reuse default renderer #"
1877                    << vie_channel_;
1878       *renderer = mux_channels_[0]->render_adapter()->renderer();
1879       return true;
1880     }
1881     return false;
1882   }
1883 
1884   *renderer = it->second->render_adapter()->renderer();
1885   return true;
1886 }
1887 
1888 // TODO: Add unittests to test this function.
SendFrame(uint32 ssrc,const VideoFrame * frame)1889 bool WebRtcVideoMediaChannel::SendFrame(uint32 ssrc, const VideoFrame* frame) {
1890   if (ssrc != 0 || !sending() || !external_capture_) {
1891     return false;
1892   }
1893 
1894   // Update local stream statistics.
1895   local_stream_info_->UpdateFrame(frame->GetWidth(), frame->GetHeight());
1896 
1897   // Checks if we need to reset vie send codec.
1898   if (!MaybeResetVieSendCodec(frame->GetWidth(), frame->GetHeight(), NULL)) {
1899     LOG(LS_ERROR) << "MaybeResetVieSendCodec failed with "
1900                   << frame->GetWidth() << "x" << frame->GetHeight();
1901     return false;
1902   }
1903 
1904   // Blacken the frame if video is muted.
1905   const VideoFrame* frame_out = frame;
1906   talk_base::scoped_ptr<VideoFrame> black_frame;
1907   if (muted_) {
1908     black_frame.reset(frame->Copy());
1909     black_frame->SetToBlack();
1910     frame_out = black_frame.get();
1911   }
1912 
1913   webrtc::ViEVideoFrameI420 frame_i420;
1914   // TODO: Update the webrtc::ViEVideoFrameI420
1915   // to use const unsigned char*
1916   frame_i420.y_plane = const_cast<unsigned char*>(frame_out->GetYPlane());
1917   frame_i420.u_plane = const_cast<unsigned char*>(frame_out->GetUPlane());
1918   frame_i420.v_plane = const_cast<unsigned char*>(frame_out->GetVPlane());
1919   frame_i420.y_pitch = frame_out->GetYPitch();
1920   frame_i420.u_pitch = frame_out->GetUPitch();
1921   frame_i420.v_pitch = frame_out->GetVPitch();
1922   frame_i420.width = frame_out->GetWidth();
1923   frame_i420.height = frame_out->GetHeight();
1924 
1925   // Convert from nanoseconds to milliseconds.
1926   WebRtc_Word64 clocks = frame_out->GetTimeStamp() /
1927       talk_base::kNumNanosecsPerMillisec;
1928 
1929   return (external_capture_->IncomingFrameI420(frame_i420, clocks) == 0);
1930 }
1931 
ConfigureChannel(int channel_id)1932 bool WebRtcVideoMediaChannel::ConfigureChannel(int channel_id) {
1933   // Register external transport.
1934   if (engine_->vie()->network()->RegisterSendTransport(
1935       channel_id, *this) != 0) {
1936     LOG_RTCERR1(RegisterSendTransport, channel_id);
1937     return false;
1938   }
1939 
1940   // Set MTU.
1941   if (engine_->vie()->network()->SetMTU(channel_id, kVideoMtu) != 0) {
1942     LOG_RTCERR2(SetMTU, channel_id, kVideoMtu);
1943     return false;
1944   }
1945   // Turn on RTCP and loss feedback reporting.
1946   if (engine()->vie()->rtp()->SetRTCPStatus(
1947       channel_id, webrtc::kRtcpCompound_RFC4585) != 0) {
1948     LOG_RTCERR2(SetRTCPStatus, channel_id, webrtc::kRtcpCompound_RFC4585);
1949     return false;
1950   }
1951   // Enable pli as key frame request method.
1952   if (engine_->vie()->rtp()->SetKeyFrameRequestMethod(
1953       channel_id, webrtc::kViEKeyFrameRequestPliRtcp) != 0) {
1954     LOG_RTCERR2(SetKeyFrameRequestMethod,
1955                 channel_id, webrtc::kViEKeyFrameRequestPliRtcp);
1956     return false;
1957   }
1958   return true;
1959 }
1960 
ConfigureReceiving(int channel_id,uint32 remote_ssrc)1961 bool WebRtcVideoMediaChannel::ConfigureReceiving(int channel_id,
1962                                                  uint32 remote_ssrc) {
1963   // Connect the voice channel, if there is one.
1964   // TODO: The A/V is synched by the receiving channel. So we need to
1965   // know the SSRC of the remote audio channel in order to fetch the correct
1966   // webrtc VoiceEngine channel. For now- only sync the default channel used
1967   // in 1-1 calls.
1968   if (remote_ssrc == 0 && voice_channel_) {
1969     WebRtcVoiceMediaChannel* voice_channel =
1970         static_cast<WebRtcVoiceMediaChannel*>(voice_channel_);
1971     if (engine_->vie()->base()->ConnectAudioChannel(
1972         vie_channel_, voice_channel->voe_channel()) != 0) {
1973       LOG_RTCERR2(ConnectAudioChannel, channel_id,
1974                   voice_channel->voe_channel());
1975       LOG(LS_WARNING) << "A/V not synchronized";
1976       // Not a fatal error.
1977     }
1978   }
1979 
1980   talk_base::scoped_ptr<WebRtcVideoChannelInfo> channel_info(
1981       new WebRtcVideoChannelInfo(channel_id));
1982 
1983   // Install a render adapter.
1984   if (engine_->vie()->render()->AddRenderer(channel_id,
1985       webrtc::kVideoI420, channel_info->render_adapter()) != 0) {
1986     LOG_RTCERR3(AddRenderer, channel_id, webrtc::kVideoI420,
1987                 channel_info->render_adapter());
1988     return false;
1989   }
1990 
1991   // TODO Change this once REMB supporting multiple sending channels.
1992   // Turn off remb sending (2nd param) and turn on remb reporting (3rd param)
1993   // here.
1994   // For sending channel, remb sending will be turned on after StartSending.
1995   if (engine_->vie()->rtp()->SetRembStatus(channel_id, false, true) != 0) {
1996     LOG_RTCERR3(SetRembStatus, vie_channel_, false, true);
1997     return false;
1998   }
1999 
2000   if (remote_ssrc != 0) {
2001     // Use the same SSRC as our default channel
2002     // (so the RTCP reports are correct).
2003     unsigned int send_ssrc = 0;
2004     webrtc::ViERTP_RTCP* rtp = engine()->vie()->rtp();
2005     if (rtp->GetLocalSSRC(vie_channel_, send_ssrc) == -1) {
2006       LOG_RTCERR2(GetSendSSRC, channel_id, send_ssrc);
2007       return false;
2008     }
2009     if (rtp->SetLocalSSRC(channel_id, send_ssrc) == -1) {
2010       LOG_RTCERR2(SetSendSSRC, channel_id, send_ssrc);
2011       return false;
2012     }
2013   }  // Else this is the the default channel and we don't change the SSRC.
2014 
2015   // Disable color enhancement since it is a bit too aggressive.
2016   if (engine()->vie()->image()->EnableColorEnhancement(channel_id,
2017                                                        false) != 0) {
2018     LOG_RTCERR1(EnableColorEnhancement, channel_id);
2019     return false;
2020   }
2021 
2022   if (!SetReceiveCodecs(channel_id)) {
2023     return false;
2024   }
2025 
2026   if (render_started_) {
2027     if (engine_->vie()->render()->StartRender(channel_id) != 0) {
2028       LOG_RTCERR1(StartRender, channel_id);
2029       return false;
2030     }
2031   }
2032 
2033   // Register decoder observer for incoming framerate and bitrate.
2034   if (engine()->vie()->codec()->RegisterDecoderObserver(
2035       channel_id, *channel_info->decoder_observer()) != 0) {
2036     LOG_RTCERR1(RegisterDecoderObserver, channel_info->decoder_observer());
2037     return false;
2038   }
2039 
2040   mux_channels_[remote_ssrc] = channel_info.release();
2041 
2042   return true;
2043 }
2044 
SetNackFec(int channel_id,int red_payload_type,int fec_payload_type)2045 bool WebRtcVideoMediaChannel::SetNackFec(int channel_id,
2046                                          int red_payload_type,
2047                                          int fec_payload_type) {
2048   // Enable hybrid NACK/FEC if negotiated and not in a conference, use only NACK
2049   // otherwise.
2050   bool enable = (red_payload_type != -1 && fec_payload_type != -1 &&
2051       !(options_ & OPT_CONFERENCE));
2052   if (enable) {
2053     if (engine_->vie()->rtp()->SetHybridNACKFECStatus(
2054         channel_id, enable, red_payload_type, fec_payload_type) != 0) {
2055       LOG_RTCERR4(SetHybridNACKFECStatus,
2056                   channel_id, enable, red_payload_type, fec_payload_type);
2057       return false;
2058     }
2059     LOG(LS_INFO) << "Hybrid NACK/FEC enabled for channel " << channel_id;
2060   } else {
2061     if (engine_->vie()->rtp()->SetNACKStatus(channel_id, true) != 0) {
2062       LOG_RTCERR1(SetNACKStatus, channel_id);
2063       return false;
2064     }
2065     LOG(LS_INFO) << "NACK enabled for channel " << channel_id;
2066   }
2067   return true;
2068 }
2069 
SetSendCodec(const webrtc::VideoCodec & codec,int min_bitrate,int start_bitrate,int max_bitrate)2070 bool WebRtcVideoMediaChannel::SetSendCodec(const webrtc::VideoCodec& codec,
2071                                            int min_bitrate,
2072                                            int start_bitrate,
2073                                            int max_bitrate) {
2074   // Make a copy of the codec
2075   webrtc::VideoCodec target_codec = codec;
2076   target_codec.startBitrate = start_bitrate;
2077   target_codec.minBitrate = min_bitrate;
2078   target_codec.maxBitrate = max_bitrate;
2079 
2080   if (engine()->vie()->codec()->SetSendCodec(vie_channel_, target_codec) != 0) {
2081     LOG_RTCERR2(SetSendCodec, vie_channel_, send_codec_->plName);
2082     return false;
2083   }
2084 
2085   // Reset the send_codec_ only if SetSendCodec is success.
2086   send_codec_.reset(new webrtc::VideoCodec(target_codec));
2087   send_min_bitrate_ = min_bitrate;
2088   send_start_bitrate_ = start_bitrate;
2089   send_max_bitrate_ = max_bitrate;
2090 
2091   return true;
2092 }
2093 
LogSendCodecChange(const std::string & reason)2094 void WebRtcVideoMediaChannel::LogSendCodecChange(const std::string& reason) {
2095   webrtc::VideoCodec vie_codec;
2096   if (engine()->vie()->codec()->GetSendCodec(vie_channel_, vie_codec) != 0) {
2097     LOG_RTCERR1(GetSendCodec, vie_channel_);
2098     return;
2099   }
2100 
2101   LOG(LS_INFO) << reason << " : selected video codec "
2102                << vie_codec.plName << "/"
2103                << vie_codec.width << "x" << vie_codec.height << "x"
2104                << static_cast<int>(vie_codec.maxFramerate) << "fps"
2105                << "@" << vie_codec.maxBitrate << "kbps";
2106   if (webrtc::kVideoCodecVP8 == vie_codec.codecType) {
2107     LOG(LS_INFO) << "VP8 number of temporal layers: "
2108                  << static_cast<int>(
2109                     vie_codec.codecSpecific.VP8.numberOfTemporalLayers);
2110   }
2111 
2112 }
2113 
SetReceiveCodecs(int channel_id)2114 bool WebRtcVideoMediaChannel::SetReceiveCodecs(int channel_id) {
2115   int red_type = -1;
2116   int fec_type = -1;
2117   for (std::vector<webrtc::VideoCodec>::iterator it = receive_codecs_.begin();
2118        it != receive_codecs_.end(); ++it) {
2119     if (it->codecType == webrtc::kVideoCodecRED) {
2120       red_type = it->plType;
2121     } else if (it->codecType == webrtc::kVideoCodecULPFEC) {
2122       fec_type = it->plType;
2123     }
2124     if (engine()->vie()->codec()->SetReceiveCodec(channel_id, *it) != 0) {
2125       LOG_RTCERR2(SetReceiveCodec, channel_id, it->plName);
2126       return false;
2127     }
2128   }
2129 
2130   // Enable video protection. For a sending channel, this will be taken care of
2131   // in SetSendCodecs.
2132   if (channel_id != vie_channel_) {
2133     if (!SetNackFec(channel_id, red_type, fec_type)) {
2134       return false;
2135     }
2136   }
2137 
2138   // Start receiving packets if at least one receive codec has been set.
2139   if (!receive_codecs_.empty()) {
2140     if (engine()->vie()->base()->StartReceive(channel_id) != 0) {
2141       LOG_RTCERR1(StartReceive, channel_id);
2142       return false;
2143     }
2144   }
2145   return true;
2146 }
2147 
2148 // If the new frame size is different from the send codec size we set on vie,
2149 // we need to reset the send codec on vie.
2150 // The new send codec size should not exceed send_codec_ which is controlled
2151 // only by the 'jec' logic.
MaybeResetVieSendCodec(int new_width,int new_height,bool * reset)2152 bool WebRtcVideoMediaChannel::MaybeResetVieSendCodec(int new_width,
2153                                                      int new_height,
2154                                                      bool* reset) {
2155   if (reset) {
2156     *reset = false;
2157   }
2158 
2159   if (NULL == send_codec_.get()) {
2160     return false;
2161   }
2162 
2163   // Vie send codec size should not exceed send_codec_.
2164   int target_width = new_width;
2165   int target_height = new_height;
2166   if (new_width > send_codec_->width || new_height > send_codec_->height) {
2167     target_width = send_codec_->width;
2168     target_height = send_codec_->height;
2169   }
2170 
2171   // Get current vie codec.
2172   webrtc::VideoCodec vie_codec;
2173   if (engine()->vie()->codec()->GetSendCodec(vie_channel_, vie_codec) != 0) {
2174     LOG_RTCERR1(GetSendCodec, vie_channel_);
2175     return false;
2176   }
2177 
2178   // Only reset send codec when there is a size change.
2179   if (target_width != vie_codec.width || target_height != vie_codec.height) {
2180     // Set the new codec on vie.
2181     vie_codec.width = target_width;
2182     vie_codec.height = target_height;
2183 
2184     if (engine()->vie()->codec()->SetSendCodec(vie_channel_, vie_codec) != 0) {
2185       LOG_RTCERR1(SetSendCodec, vie_channel_);
2186       return false;
2187     }
2188     if (reset) {
2189       *reset = true;
2190     }
2191     LogSendCodecChange("Capture size changed");
2192   }
2193 
2194   return true;
2195 }
2196 
SendPacket(int channel,const void * data,int len)2197 int WebRtcVideoMediaChannel::SendPacket(int channel, const void* data,
2198                                         int len) {
2199   if (!network_interface_) {
2200     return -1;
2201   }
2202   talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
2203   return network_interface_->SendPacket(&packet) ? len : -1;
2204 }
2205 
SendRTCPPacket(int channel,const void * data,int len)2206 int WebRtcVideoMediaChannel::SendRTCPPacket(int channel,
2207                                             const void* data,
2208                                             int len) {
2209   if (!network_interface_) {
2210     return -1;
2211   }
2212   talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
2213   return network_interface_->SendRtcp(&packet) ? len : -1;
2214 }
2215 
2216 }  // namespace cricket
2217 
2218 #endif  // HAVE_WEBRTC_VIDEO
2219