1 /*
2 * libjingle
3 * Copyright 2004--2011, Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31
32 #ifdef HAVE_WEBRTC_VIDEO
33
34 #include "talk/session/phone/webrtcvideoengine.h"
35
36 #include "talk/base/basictypes.h"
37 #include "talk/base/common.h"
38 #include "talk/base/buffer.h"
39 #include "talk/base/byteorder.h"
40 #include "talk/base/logging.h"
41 #include "talk/base/stringutils.h"
42 #include "talk/session/phone/filevideocapturer.h"
43 #include "talk/session/phone/rtputils.h"
44 #include "talk/session/phone/streamparams.h"
45 #include "talk/session/phone/videorenderer.h"
46 #include "talk/session/phone/webrtcpassthroughrender.h"
47 #include "talk/session/phone/webrtcvoiceengine.h"
48 #include "talk/session/phone/webrtcvideocapturer.h"
49 #include "talk/session/phone/webrtcvideoframe.h"
50 #include "talk/session/phone/webrtcvie.h"
51 #include "talk/session/phone/webrtcvoe.h"
52
53 namespace cricket {
54
55 static const int kDefaultLogSeverity = talk_base::LS_WARNING;
56
57 static const int kMinVideoBitrate = 100;
58 static const int kStartVideoBitrate = 300;
59 static const int kMaxVideoBitrate = 2000;
60 static const int kDefaultConferenceModeMaxVideoBitrate = 500;
61
62 static const int kVideoMtu = 1200;
63
64 static const int kVideoRtpBufferSize = 65536;
65
66 static const char kVp8PayloadName[] = "VP8";
67 static const char kRedPayloadName[] = "red";
68 static const char kFecPayloadName[] = "ulpfec";
69
70 static const int kDefaultNumberOfTemporalLayers = 3;
71
LogMultiline(talk_base::LoggingSeverity sev,char * text)72 static void LogMultiline(talk_base::LoggingSeverity sev, char* text) {
73 const char* delim = "\r\n";
74 for (char* tok = strtok(text, delim); tok; tok = strtok(NULL, delim)) {
75 LOG_V(sev) << tok;
76 }
77 }
78
79 class WebRtcRenderAdapter : public webrtc::ExternalRenderer {
80 public:
WebRtcRenderAdapter(VideoRenderer * renderer)81 explicit WebRtcRenderAdapter(VideoRenderer* renderer)
82 : renderer_(renderer), width_(0), height_(0) {
83 }
~WebRtcRenderAdapter()84 virtual ~WebRtcRenderAdapter() {
85 }
86
SetRenderer(VideoRenderer * renderer)87 void SetRenderer(VideoRenderer* renderer) {
88 talk_base::CritScope cs(&crit_);
89 renderer_ = renderer;
90 // FrameSizeChange may have already been called when renderer was not set.
91 // If so we should call SetSize here.
92 // TODO: Add unit test for this case. Didn't do it now
93 // because the WebRtcRenderAdapter is currently hiding in cc file. No
94 // good way to get access to it from the unit test.
95 if (width_ > 0 && height_ > 0 && renderer_ != NULL) {
96 if (!renderer_->SetSize(width_, height_, 0)) {
97 LOG(LS_ERROR)
98 << "WebRtcRenderAdapter SetRenderer failed to SetSize to: "
99 << width_ << "x" << height_;
100 }
101 }
102 }
103 // Implementation of webrtc::ExternalRenderer.
FrameSizeChange(unsigned int width,unsigned int height,unsigned int)104 virtual int FrameSizeChange(unsigned int width, unsigned int height,
105 unsigned int /*number_of_streams*/) {
106 talk_base::CritScope cs(&crit_);
107 width_ = width;
108 height_ = height;
109 LOG(LS_INFO) << "WebRtcRenderAdapter frame size changed to: "
110 << width << "x" << height;
111 if (renderer_ == NULL) {
112 LOG(LS_VERBOSE) << "WebRtcRenderAdapter the renderer has not been set. "
113 << "SetSize will be called later in SetRenderer.";
114 return 0;
115 }
116 return renderer_->SetSize(width_, height_, 0) ? 0 : -1;
117 }
DeliverFrame(unsigned char * buffer,int buffer_size,uint32_t time_stamp,int64_t render_time)118 virtual int DeliverFrame(unsigned char* buffer, int buffer_size,
119 uint32_t time_stamp, int64_t render_time) {
120 talk_base::CritScope cs(&crit_);
121 frame_rate_tracker_.Update(1);
122 if (renderer_ == NULL) {
123 return 0;
124 }
125 WebRtcVideoFrame video_frame;
126 // Convert 90K rtp timestamp to ns timestamp.
127 int64 rtp_time_stamp_in_ns = (time_stamp / 90) *
128 talk_base::kNumNanosecsPerMillisec;
129 // Convert milisecond render time to ns timestamp.
130 int64 render_time_stamp_in_ns = render_time *
131 talk_base::kNumNanosecsPerMillisec;
132 // Send the rtp timestamp to renderer as the VideoFrame timestamp.
133 // and the render timestamp as the VideoFrame elapsed_time.
134 video_frame.Attach(buffer, buffer_size, width_, height_,
135 1, 1, render_time_stamp_in_ns,
136 rtp_time_stamp_in_ns, 0);
137
138 // Sanity check on decoded frame size.
139 if (buffer_size != static_cast<int>(VideoFrame::SizeOf(width_, height_))) {
140 LOG(LS_WARNING) << "WebRtcRenderAdapter received a strange frame size: "
141 << buffer_size;
142 }
143
144 int ret = renderer_->RenderFrame(&video_frame) ? 0 : -1;
145 uint8* buffer_temp;
146 size_t buffer_size_temp;
147 video_frame.Detach(&buffer_temp, &buffer_size_temp);
148 return ret;
149 }
150
width()151 unsigned int width() {
152 talk_base::CritScope cs(&crit_);
153 return width_;
154 }
height()155 unsigned int height() {
156 talk_base::CritScope cs(&crit_);
157 return height_;
158 }
framerate()159 int framerate() {
160 talk_base::CritScope cs(&crit_);
161 return frame_rate_tracker_.units_second();
162 }
renderer()163 VideoRenderer* renderer() {
164 talk_base::CritScope cs(&crit_);
165 return renderer_;
166 }
167
168 private:
169 talk_base::CriticalSection crit_;
170 VideoRenderer* renderer_;
171 unsigned int width_;
172 unsigned int height_;
173 talk_base::RateTracker frame_rate_tracker_;
174 };
175
176 class WebRtcDecoderObserver : public webrtc::ViEDecoderObserver {
177 public:
WebRtcDecoderObserver(int video_channel)178 explicit WebRtcDecoderObserver(int video_channel)
179 : video_channel_(video_channel),
180 framerate_(0),
181 bitrate_(0),
182 firs_requested_(0) {
183 }
184
185 // virtual functions from VieDecoderObserver.
IncomingCodecChanged(const int videoChannel,const webrtc::VideoCodec & videoCodec)186 virtual void IncomingCodecChanged(const int videoChannel,
187 const webrtc::VideoCodec& videoCodec) {}
IncomingRate(const int videoChannel,const unsigned int framerate,const unsigned int bitrate)188 virtual void IncomingRate(const int videoChannel,
189 const unsigned int framerate,
190 const unsigned int bitrate) {
191 ASSERT(video_channel_ == videoChannel);
192 framerate_ = framerate;
193 bitrate_ = bitrate;
194 }
RequestNewKeyFrame(const int videoChannel)195 virtual void RequestNewKeyFrame(const int videoChannel) {
196 ASSERT(video_channel_ == videoChannel);
197 ++firs_requested_;
198 }
199
framerate() const200 int framerate() const { return framerate_; }
bitrate() const201 int bitrate() const { return bitrate_; }
firs_requested() const202 int firs_requested() const { return firs_requested_; }
203
204 private:
205 int video_channel_;
206 int framerate_;
207 int bitrate_;
208 int firs_requested_;
209 };
210
211 class WebRtcEncoderObserver : public webrtc::ViEEncoderObserver {
212 public:
WebRtcEncoderObserver(int video_channel)213 explicit WebRtcEncoderObserver(int video_channel)
214 : video_channel_(video_channel),
215 framerate_(0),
216 bitrate_(0) {
217 }
218
219 // virtual functions from VieEncoderObserver.
OutgoingRate(const int videoChannel,const unsigned int framerate,const unsigned int bitrate)220 virtual void OutgoingRate(const int videoChannel,
221 const unsigned int framerate,
222 const unsigned int bitrate) {
223 ASSERT(video_channel_ == videoChannel);
224 framerate_ = framerate;
225 bitrate_ = bitrate;
226 }
227
framerate() const228 int framerate() const { return framerate_; }
bitrate() const229 int bitrate() const { return bitrate_; }
230
231 private:
232 int video_channel_;
233 int framerate_;
234 int bitrate_;
235 };
236
237 class WebRtcLocalStreamInfo {
238 public:
width()239 int width() {
240 talk_base::CritScope cs(&crit_);
241 return width_;
242 }
height()243 int height() {
244 talk_base::CritScope cs(&crit_);
245 return height_;
246 }
framerate()247 int framerate() {
248 talk_base::CritScope cs(&crit_);
249 return rate_tracker_.units_second();
250 }
251
UpdateFrame(int width,int height)252 void UpdateFrame(int width, int height) {
253 talk_base::CritScope cs(&crit_);
254 width_ = width;
255 height_ = height;
256 rate_tracker_.Update(1);
257 }
258
259 private:
260 talk_base::CriticalSection crit_;
261 unsigned int width_;
262 unsigned int height_;
263 talk_base::RateTracker rate_tracker_;
264 };
265
266 // WebRtcVideoChannelInfo is a container class with members such as renderer
267 // and a decoder observer that is used by receive channels.
268 // It must exist as long as the receive channel is connected to renderer or a
269 // decoder observer in this class and methods in the class should only be called
270 // from the worker thread.
271 class WebRtcVideoChannelInfo {
272 public:
WebRtcVideoChannelInfo(int channel_id)273 explicit WebRtcVideoChannelInfo(int channel_id)
274 : channel_id_(channel_id),
275 render_adapter_(NULL),
276 decoder_observer_(channel_id) {
277 }
channel_id()278 int channel_id() { return channel_id_; }
SetRenderer(VideoRenderer * renderer)279 void SetRenderer(VideoRenderer* renderer) {
280 render_adapter_.SetRenderer(renderer);
281 }
render_adapter()282 WebRtcRenderAdapter* render_adapter() { return &render_adapter_; }
decoder_observer()283 WebRtcDecoderObserver* decoder_observer() { return &decoder_observer_; }
284
285 private:
286 int channel_id_; // Webrtc video channel number.
287 // Renderer for this channel.
288 WebRtcRenderAdapter render_adapter_;
289 WebRtcDecoderObserver decoder_observer_;
290 };
291
292 const WebRtcVideoEngine::VideoCodecPref
293 WebRtcVideoEngine::kVideoCodecPrefs[] = {
294 {kVp8PayloadName, 100, 0},
295 {kRedPayloadName, 101, 1},
296 {kFecPayloadName, 102, 2},
297 };
298
299 static const int64 kNsPerFrame = 33333333; // 30fps
300
301 // The formats are sorted by the descending order of width. We use the order to
302 // find the next format for CPU and bandwidth adaptation.
303 const VideoFormatPod WebRtcVideoEngine::kVideoFormats[] = {
304 {1280, 800, kNsPerFrame, FOURCC_ANY},
305 {1280, 720, kNsPerFrame, FOURCC_ANY},
306 {960, 600, kNsPerFrame, FOURCC_ANY},
307 {960, 540, kNsPerFrame, FOURCC_ANY},
308 {640, 400, kNsPerFrame, FOURCC_ANY},
309 {640, 360, kNsPerFrame, FOURCC_ANY},
310 {640, 480, kNsPerFrame, FOURCC_ANY},
311 {480, 300, kNsPerFrame, FOURCC_ANY},
312 {480, 270, kNsPerFrame, FOURCC_ANY},
313 {480, 360, kNsPerFrame, FOURCC_ANY},
314 {320, 200, kNsPerFrame, FOURCC_ANY},
315 {320, 180, kNsPerFrame, FOURCC_ANY},
316 {320, 240, kNsPerFrame, FOURCC_ANY},
317 {240, 150, kNsPerFrame, FOURCC_ANY},
318 {240, 135, kNsPerFrame, FOURCC_ANY},
319 {240, 180, kNsPerFrame, FOURCC_ANY},
320 {160, 100, kNsPerFrame, FOURCC_ANY},
321 {160, 90, kNsPerFrame, FOURCC_ANY},
322 {160, 120, kNsPerFrame, FOURCC_ANY},
323 };
324
325 const VideoFormatPod WebRtcVideoEngine::kDefaultVideoFormat =
326 {640, 400, kNsPerFrame, FOURCC_ANY};
327
WebRtcVideoEngine()328 WebRtcVideoEngine::WebRtcVideoEngine() {
329 Construct(new ViEWrapper(), new ViETraceWrapper(), NULL);
330 }
331
WebRtcVideoEngine(WebRtcVoiceEngine * voice_engine,ViEWrapper * vie_wrapper)332 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
333 ViEWrapper* vie_wrapper) {
334 Construct(vie_wrapper, new ViETraceWrapper(), voice_engine);
335 }
336
WebRtcVideoEngine(WebRtcVoiceEngine * voice_engine,ViEWrapper * vie_wrapper,ViETraceWrapper * tracing)337 WebRtcVideoEngine::WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
338 ViEWrapper* vie_wrapper,
339 ViETraceWrapper* tracing) {
340 Construct(vie_wrapper, tracing, voice_engine);
341 }
342
Construct(ViEWrapper * vie_wrapper,ViETraceWrapper * tracing,WebRtcVoiceEngine * voice_engine)343 void WebRtcVideoEngine::Construct(ViEWrapper* vie_wrapper,
344 ViETraceWrapper* tracing,
345 WebRtcVoiceEngine* voice_engine) {
346 LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine";
347 vie_wrapper_.reset(vie_wrapper);
348 vie_wrapper_base_initialized_ = false;
349 tracing_.reset(tracing);
350 voice_engine_ = voice_engine;
351 initialized_ = false;
352 log_level_ = kDefaultLogSeverity;
353 render_module_.reset(new WebRtcPassthroughRender());
354 local_renderer_w_ = local_renderer_h_ = 0;
355 local_renderer_ = NULL;
356 owns_capturer_ = false;
357 video_capturer_ = NULL;
358 capture_started_ = false;
359
360 ApplyLogging("");
361 if (tracing_->SetTraceCallback(this) != 0) {
362 LOG_RTCERR1(SetTraceCallback, this);
363 }
364
365 // Set default quality levels for our supported codecs. We override them here
366 // if we know your cpu performance is low, and they can be updated explicitly
367 // by calling SetDefaultCodec. For example by a flute preference setting, or
368 // by the server with a jec in response to our reported system info.
369 VideoCodec max_codec(kVideoCodecPrefs[0].payload_type,
370 kVideoCodecPrefs[0].name,
371 kDefaultVideoFormat.width,
372 kDefaultVideoFormat.height,
373 VideoFormat::IntervalToFps(kDefaultVideoFormat.interval),
374 0);
375 if (!SetDefaultCodec(max_codec)) {
376 LOG(LS_ERROR) << "Failed to initialize list of supported codec types";
377 }
378 }
379
~WebRtcVideoEngine()380 WebRtcVideoEngine::~WebRtcVideoEngine() {
381 ClearCapturer();
382 LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
383 if (initialized_) {
384 Terminate();
385 }
386 tracing_->SetTraceCallback(NULL);
387 // Test to see if the media processor was deregistered properly.
388 ASSERT(SignalMediaFrame.is_empty());
389 }
390
Init()391 bool WebRtcVideoEngine::Init() {
392 LOG(LS_INFO) << "WebRtcVideoEngine::Init";
393 bool result = InitVideoEngine();
394 if (result) {
395 LOG(LS_INFO) << "VideoEngine Init done";
396 } else {
397 LOG(LS_ERROR) << "VideoEngine Init failed, releasing";
398 Terminate();
399 }
400 return result;
401 }
402
InitVideoEngine()403 bool WebRtcVideoEngine::InitVideoEngine() {
404 LOG(LS_INFO) << "WebRtcVideoEngine::InitVideoEngine";
405
406 // Init WebRTC VideoEngine.
407 if (!vie_wrapper_base_initialized_) {
408 if (vie_wrapper_->base()->Init() != 0) {
409 LOG_RTCERR0(Init);
410 return false;
411 }
412 vie_wrapper_base_initialized_ = true;
413 }
414
415 // Log the VoiceEngine version info.
416 char buffer[1024] = "";
417 if (vie_wrapper_->base()->GetVersion(buffer) != 0) {
418 LOG_RTCERR0(GetVersion);
419 return false;
420 }
421
422 LOG(LS_INFO) << "WebRtc VideoEngine Version:";
423 LogMultiline(talk_base::LS_INFO, buffer);
424
425 // Hook up to VoiceEngine for sync purposes, if supplied.
426 if (!voice_engine_) {
427 LOG(LS_WARNING) << "NULL voice engine";
428 } else if ((vie_wrapper_->base()->SetVoiceEngine(
429 voice_engine_->voe()->engine())) != 0) {
430 LOG_RTCERR0(SetVoiceEngine);
431 return false;
432 }
433
434 // Register for callbacks from the engine.
435 if ((vie_wrapper_->base()->RegisterObserver(*this)) != 0) {
436 LOG_RTCERR0(RegisterObserver);
437 return false;
438 }
439
440 // Register our custom render module.
441 if (vie_wrapper_->render()->RegisterVideoRenderModule(
442 *render_module_.get()) != 0) {
443 LOG_RTCERR0(RegisterVideoRenderModule);
444 return false;
445 }
446
447 initialized_ = true;
448 return true;
449 }
450
Terminate()451 void WebRtcVideoEngine::Terminate() {
452 LOG(LS_INFO) << "WebRtcVideoEngine::Terminate";
453 initialized_ = false;
454 SetCapture(false);
455
456 if (vie_wrapper_->render()->DeRegisterVideoRenderModule(
457 *render_module_.get()) != 0) {
458 LOG_RTCERR0(DeRegisterVideoRenderModule);
459 }
460
461 if (vie_wrapper_->base()->DeregisterObserver() != 0) {
462 LOG_RTCERR0(DeregisterObserver);
463 }
464
465 if (vie_wrapper_->base()->SetVoiceEngine(NULL) != 0) {
466 LOG_RTCERR0(SetVoiceEngine);
467 }
468 }
469
GetCapabilities()470 int WebRtcVideoEngine::GetCapabilities() {
471 return VIDEO_RECV | VIDEO_SEND;
472 }
473
SetOptions(int options)474 bool WebRtcVideoEngine::SetOptions(int options) {
475 return true;
476 }
477
SetDefaultEncoderConfig(const VideoEncoderConfig & config)478 bool WebRtcVideoEngine::SetDefaultEncoderConfig(
479 const VideoEncoderConfig& config) {
480 return SetDefaultCodec(config.max_codec);
481 }
482
483 // SetDefaultCodec may be called while the capturer is running. For example, a
484 // test call is started in a page with QVGA default codec, and then a real call
485 // is started in another page with VGA default codec. This is the corner case
486 // and happens only when a session is started. We ignore this case currently.
SetDefaultCodec(const VideoCodec & codec)487 bool WebRtcVideoEngine::SetDefaultCodec(const VideoCodec& codec) {
488 if (!RebuildCodecList(codec)) {
489 LOG(LS_WARNING) << "Failed to RebuildCodecList";
490 return false;
491 }
492
493 default_codec_format_ = VideoFormat(
494 video_codecs_[0].width,
495 video_codecs_[0].height,
496 VideoFormat::FpsToInterval(video_codecs_[0].framerate),
497 FOURCC_ANY);
498 return true;
499 }
500
CreateChannel(VoiceMediaChannel * voice_channel)501 WebRtcVideoMediaChannel* WebRtcVideoEngine::CreateChannel(
502 VoiceMediaChannel* voice_channel) {
503 WebRtcVideoMediaChannel* channel =
504 new WebRtcVideoMediaChannel(this, voice_channel);
505 if (!channel->Init()) {
506 delete channel;
507 channel = NULL;
508 }
509 return channel;
510 }
511
SetCaptureDevice(const Device * device)512 bool WebRtcVideoEngine::SetCaptureDevice(const Device* device) {
513 const bool owns_capturer = true;
514 if (!device) {
515 if (!SetCapturer(NULL, owns_capturer)) {
516 return false;
517 }
518 LOG(LS_INFO) << "Camera set to NULL";
519 return true;
520 }
521 // No-op if the device hasn't changed.
522 if ((video_capturer_ != NULL) && video_capturer_->GetId() == device->id) {
523 return true;
524 }
525 // Create a new capturer for the specified device.
526 VideoCapturer* capturer = CreateVideoCapturer(*device);
527 if (!capturer) {
528 LOG(LS_ERROR) << "Failed to create camera '" << device->name << "', id='"
529 << device->id << "'";
530 return false;
531 }
532 if (!SetCapturer(capturer, owns_capturer)) {
533 return false;
534 }
535 LOG(LS_INFO) << "Camera set to '" << device->name << "', id='"
536 << device->id << "'";
537 return true;
538 }
539
SetCaptureModule(webrtc::VideoCaptureModule * vcm)540 bool WebRtcVideoEngine::SetCaptureModule(webrtc::VideoCaptureModule* vcm) {
541 const bool owns_capturer = true;
542 if (!vcm) {
543 if (!SetCapturer(NULL, owns_capturer)) {
544 return false;
545 }
546 LOG(LS_INFO) << "Camera set to NULL";
547 return true;
548 }
549 // Create a new capturer for the specified device.
550 WebRtcVideoCapturer* capturer = new WebRtcVideoCapturer;
551 if (!capturer->Init(vcm)) {
552 LOG(LS_ERROR) << "Failed to create camera from VCM";
553 delete capturer;
554 return false;
555 }
556 if (!SetCapturer(capturer, owns_capturer)) {
557 return false;
558 }
559 LOG(LS_INFO) << "Camera created with VCM";
560 CaptureResult ret = SetCapture(true);
561 if (ret != cricket::CR_SUCCESS && ret != cricket::CR_PENDING) {
562 LOG(LS_ERROR) << "Failed to start camera.";
563 return false;
564 }
565 return true;
566 }
567
SetVideoCapturer(VideoCapturer * capturer,uint32)568 bool WebRtcVideoEngine::SetVideoCapturer(VideoCapturer* capturer,
569 uint32 /*ssrc*/) {
570 const bool owns_capturer = false;
571 return SetCapturer(capturer, owns_capturer);
572 }
573
SetLocalRenderer(VideoRenderer * renderer)574 bool WebRtcVideoEngine::SetLocalRenderer(VideoRenderer* renderer) {
575 local_renderer_w_ = local_renderer_h_ = 0;
576 local_renderer_ = renderer;
577 return true;
578 }
579
SetCapture(bool capture)580 CaptureResult WebRtcVideoEngine::SetCapture(bool capture) {
581 bool old_capture = capture_started_;
582 capture_started_ = capture;
583 CaptureResult res = UpdateCapturingState();
584 if (res != CR_SUCCESS && res != CR_PENDING) {
585 capture_started_ = old_capture;
586 }
587 return res;
588 }
589
CreateVideoCapturer(const Device & device)590 VideoCapturer* WebRtcVideoEngine::CreateVideoCapturer(const Device& device) {
591 if (FileVideoCapturer::IsFileVideoCapturerDevice(device)) {
592 FileVideoCapturer* capturer = new FileVideoCapturer;
593 if (!capturer->Init(device)) {
594 delete capturer;
595 return NULL;
596 }
597 capturer->set_repeat(talk_base::kForever);
598 return capturer;
599 } else {
600 WebRtcVideoCapturer* capturer = new WebRtcVideoCapturer;
601 if (!capturer->Init(device)) {
602 delete capturer;
603 return NULL;
604 }
605 return capturer;
606 }
607 }
608
UpdateCapturingState()609 CaptureResult WebRtcVideoEngine::UpdateCapturingState() {
610 CaptureResult result = CR_SUCCESS;
611
612 bool capture = capture_started_;
613 if (!IsCapturing() && capture) { // Start capturing.
614 if (video_capturer_ == NULL) {
615 return CR_NO_DEVICE;
616 }
617
618 VideoFormat capture_format;
619 if (!video_capturer_->GetBestCaptureFormat(default_codec_format_,
620 &capture_format)) {
621 LOG(LS_WARNING) << "Unsupported format:"
622 << " width=" << default_codec_format_.width
623 << " height=" << default_codec_format_.height
624 << ". Supported formats are:";
625 const std::vector<VideoFormat>* formats =
626 video_capturer_->GetSupportedFormats();
627 if (formats) {
628 for (std::vector<VideoFormat>::const_iterator i = formats->begin();
629 i != formats->end(); ++i) {
630 const VideoFormat& format = *i;
631 LOG(LS_WARNING) << " " << GetFourccName(format.fourcc) << ":"
632 << format.width << "x" << format.height << "x"
633 << format.framerate();
634 }
635 }
636 return CR_FAILURE;
637 }
638
639 // Start the video capturer.
640 result = video_capturer_->Start(capture_format);
641 if (CR_SUCCESS != result && CR_PENDING != result) {
642 LOG(LS_ERROR) << "Failed to start the video capturer";
643 return result;
644 }
645 } else if (IsCapturing() && !capture) { // Stop capturing.
646 video_capturer_->Stop();
647 }
648
649 return result;
650 }
651
IsCapturing() const652 bool WebRtcVideoEngine::IsCapturing() const {
653 return (video_capturer_ != NULL) && video_capturer_->IsRunning();
654 }
655
OnFrameCaptured(VideoCapturer * capturer,const CapturedFrame * frame)656 void WebRtcVideoEngine::OnFrameCaptured(VideoCapturer* capturer,
657 const CapturedFrame* frame) {
658 // Force 16:10 for now. We'll be smarter with the capture refactor.
659 int cropped_height = frame->width * default_codec_format_.height
660 / default_codec_format_.width;
661 if (cropped_height > frame->height) {
662 // TODO: Once we support horizontal cropping, add cropped_width.
663 cropped_height = frame->height;
664 }
665
666 // This CapturedFrame* will already be in I420. In the future, when
667 // WebRtcVideoFrame has support for independent planes, we can just attach
668 // to it and update the pointers when cropping.
669 WebRtcVideoFrame i420_frame;
670 if (!i420_frame.Init(frame, frame->width, cropped_height)) {
671 LOG(LS_ERROR) << "Couldn't convert to I420! "
672 << frame->width << " x " << cropped_height;
673 return;
674 }
675
676 // TODO: This is the trigger point for Tx video processing.
677 // Once the capturer refactoring is done, we will move this into the
678 // capturer...it's not there right now because that image is in not in the
679 // I420 color space.
680 // The clients that subscribe will obtain meta info from the frame.
681 // When this trigger is switched over to capturer, need to pass in the real
682 // ssrc.
683 {
684 talk_base::CritScope cs(&signal_media_critical_);
685 SignalMediaFrame(kDummyVideoSsrc, &i420_frame);
686 }
687
688 // Send I420 frame to the local renderer.
689 if (local_renderer_) {
690 if (local_renderer_w_ != static_cast<int>(i420_frame.GetWidth()) ||
691 local_renderer_h_ != static_cast<int>(i420_frame.GetHeight())) {
692 local_renderer_->SetSize(local_renderer_w_ = i420_frame.GetWidth(),
693 local_renderer_h_ = i420_frame.GetHeight(), 0);
694 }
695 local_renderer_->RenderFrame(&i420_frame);
696 }
697
698 // Send I420 frame to the registered senders.
699 talk_base::CritScope cs(&channels_crit_);
700 for (VideoChannels::iterator it = channels_.begin();
701 it != channels_.end(); ++it) {
702 if ((*it)->sending()) (*it)->SendFrame(0, &i420_frame);
703 }
704 }
705
codecs() const706 const std::vector<VideoCodec>& WebRtcVideoEngine::codecs() const {
707 return video_codecs_;
708 }
709
SetLogging(int min_sev,const char * filter)710 void WebRtcVideoEngine::SetLogging(int min_sev, const char* filter) {
711 // if min_sev == -1, we keep the current log level.
712 if (min_sev >= 0) {
713 log_level_ = min_sev;
714 }
715 ApplyLogging(filter);
716 }
717
GetLastEngineError()718 int WebRtcVideoEngine::GetLastEngineError() {
719 return vie_wrapper_->error();
720 }
721
722 // Checks to see whether we comprehend and could receive a particular codec
FindCodec(const VideoCodec & in)723 bool WebRtcVideoEngine::FindCodec(const VideoCodec& in) {
724 for (int i = 0; i < ARRAY_SIZE(kVideoFormats); ++i) {
725 const VideoFormat fmt(kVideoFormats[i]);
726 if ((in.width == 0 && in.height == 0) ||
727 (fmt.width == in.width && fmt.height == in.height)) {
728 for (int j = 0; j < ARRAY_SIZE(kVideoCodecPrefs); ++j) {
729 VideoCodec codec(kVideoCodecPrefs[j].payload_type,
730 kVideoCodecPrefs[j].name, 0, 0, 0, 0);
731 if (codec.Matches(in)) {
732 return true;
733 }
734 }
735 }
736 }
737 return false;
738 }
739
740 // Given the requested codec, returns true if we can send that codec type and
741 // updates out with the best quality we could send for that codec. If current is
742 // not empty, we constrain out so that its aspect ratio matches current's.
CanSendCodec(const VideoCodec & requested,const VideoCodec & current,VideoCodec * out)743 bool WebRtcVideoEngine::CanSendCodec(const VideoCodec& requested,
744 const VideoCodec& current,
745 VideoCodec* out) {
746 if (!out) {
747 return false;
748 }
749
750 std::vector<VideoCodec>::const_iterator local_max;
751 for (local_max = video_codecs_.begin();
752 local_max < video_codecs_.end();
753 ++local_max) {
754 // First match codecs by payload type
755 if (!requested.Matches(local_max->id, local_max->name)) {
756 continue;
757 }
758
759 out->id = requested.id;
760 out->name = requested.name;
761 out->preference = requested.preference;
762 out->framerate = talk_base::_min(requested.framerate, local_max->framerate);
763 out->width = 0;
764 out->height = 0;
765
766 if (0 == requested.width && 0 == requested.height) {
767 // Special case with resolution 0. The channel should not send frames.
768 return true;
769 } else if (0 == requested.width || 0 == requested.height) {
770 // 0xn and nx0 are invalid resolutions.
771 return false;
772 }
773
774 // Pick the best quality that is within their and our bounds and has the
775 // correct aspect ratio.
776 for (int j = 0; j < ARRAY_SIZE(kVideoFormats); ++j) {
777 const VideoFormat format(kVideoFormats[j]);
778
779 // Skip any format that is larger than the local or remote maximums, or
780 // smaller than the current best match
781 if (format.width > requested.width || format.height > requested.height ||
782 format.width > local_max->width ||
783 (format.width < out->width && format.height < out->height)) {
784 continue;
785 }
786
787 bool better = false;
788
789 // Check any further constraints on this prospective format
790 if (!out->width || !out->height) {
791 // If we don't have any matches yet, this is the best so far.
792 better = true;
793 } else if (current.width && current.height) {
794 // current is set so format must match its ratio exactly.
795 better =
796 (format.width * current.height == format.height * current.width);
797 } else {
798 // Prefer closer aspect ratios i.e
799 // format.aspect - requested.aspect < out.aspect - requested.aspect
800 better = abs(format.width * requested.height * out->height -
801 requested.width * format.height * out->height) <
802 abs(out->width * format.height * requested.height -
803 requested.width * format.height * out->height);
804 }
805
806 if (better) {
807 out->width = format.width;
808 out->height = format.height;
809 }
810 }
811 if (out->width > 0) {
812 return true;
813 }
814 }
815 return false;
816 }
817
ConvertToCricketVideoCodec(const webrtc::VideoCodec & in_codec,VideoCodec * out_codec)818 void WebRtcVideoEngine::ConvertToCricketVideoCodec(
819 const webrtc::VideoCodec& in_codec, VideoCodec* out_codec) {
820 out_codec->id = in_codec.plType;
821 out_codec->name = in_codec.plName;
822 out_codec->width = in_codec.width;
823 out_codec->height = in_codec.height;
824 out_codec->framerate = in_codec.maxFramerate;
825 }
826
ConvertFromCricketVideoCodec(const VideoCodec & in_codec,webrtc::VideoCodec * out_codec)827 bool WebRtcVideoEngine::ConvertFromCricketVideoCodec(
828 const VideoCodec& in_codec, webrtc::VideoCodec* out_codec) {
829 bool found = false;
830 int ncodecs = vie_wrapper_->codec()->NumberOfCodecs();
831 for (int i = 0; i < ncodecs; ++i) {
832 if (vie_wrapper_->codec()->GetCodec(i, *out_codec) == 0 &&
833 _stricmp(in_codec.name.c_str(), out_codec->plName) == 0) {
834 found = true;
835 break;
836 }
837 }
838
839 if (!found) {
840 LOG(LS_ERROR) << "invalid codec type";
841 return false;
842 }
843
844 if (in_codec.id != 0)
845 out_codec->plType = in_codec.id;
846
847 if (in_codec.width != 0)
848 out_codec->width = in_codec.width;
849
850 if (in_codec.height != 0)
851 out_codec->height = in_codec.height;
852
853 if (in_codec.framerate != 0)
854 out_codec->maxFramerate = in_codec.framerate;
855
856 // Init the codec with the default bandwidth options.
857 out_codec->minBitrate = kMinVideoBitrate;
858 out_codec->startBitrate = kStartVideoBitrate;
859 out_codec->maxBitrate = kMaxVideoBitrate;
860
861 return true;
862 }
863
RegisterChannel(WebRtcVideoMediaChannel * channel)864 void WebRtcVideoEngine::RegisterChannel(WebRtcVideoMediaChannel *channel) {
865 talk_base::CritScope cs(&channels_crit_);
866 channels_.push_back(channel);
867 }
868
UnregisterChannel(WebRtcVideoMediaChannel * channel)869 void WebRtcVideoEngine::UnregisterChannel(WebRtcVideoMediaChannel *channel) {
870 talk_base::CritScope cs(&channels_crit_);
871 channels_.erase(std::remove(channels_.begin(), channels_.end(), channel),
872 channels_.end());
873 }
874
SetVoiceEngine(WebRtcVoiceEngine * voice_engine)875 bool WebRtcVideoEngine::SetVoiceEngine(WebRtcVoiceEngine* voice_engine) {
876 if (initialized_) {
877 LOG(LS_WARNING) << "SetVoiceEngine can not be called after Init";
878 return false;
879 }
880 voice_engine_ = voice_engine;
881 return true;
882 }
883
EnableTimedRender()884 bool WebRtcVideoEngine::EnableTimedRender() {
885 if (initialized_) {
886 LOG(LS_WARNING) << "EnableTimedRender can not be called after Init";
887 return false;
888 }
889 render_module_.reset(webrtc::VideoRender::CreateVideoRender(0, NULL,
890 false, webrtc::kRenderExternal));
891 return true;
892 }
893
894 // See https://sites.google.com/a/google.com/wavelet/
895 // Home/Magic-Flute--RTC-Engine-/Magic-Flute-Command-Line-Parameters
896 // for all supported command line setttings.
ApplyLogging(const std::string & log_filter)897 void WebRtcVideoEngine::ApplyLogging(const std::string& log_filter) {
898 int filter = 0;
899 switch (log_level_) {
900 case talk_base::LS_VERBOSE: filter |= webrtc::kTraceAll;
901 case talk_base::LS_INFO: filter |= webrtc::kTraceStateInfo;
902 case talk_base::LS_WARNING: filter |= webrtc::kTraceWarning;
903 case talk_base::LS_ERROR: filter |=
904 webrtc::kTraceError | webrtc::kTraceCritical;
905 }
906 tracing_->SetTraceFilter(filter);
907
908 // Set WebRTC trace file.
909 std::vector<std::string> opts;
910 talk_base::tokenize(log_filter, ' ', '"', '"', &opts);
911 std::vector<std::string>::iterator tracefile =
912 std::find(opts.begin(), opts.end(), "tracefile");
913 if (tracefile != opts.end() && ++tracefile != opts.end()) {
914 // Write WebRTC debug output (at same loglevel) to file
915 if (tracing_->SetTraceFile(tracefile->c_str()) == -1) {
916 LOG_RTCERR1(SetTraceFile, *tracefile);
917 }
918 }
919 }
920
921 // Rebuilds the codec list to be only those that are less intensive
922 // than the specified codec.
RebuildCodecList(const VideoCodec & in_codec)923 bool WebRtcVideoEngine::RebuildCodecList(const VideoCodec& in_codec) {
924 if (!FindCodec(in_codec))
925 return false;
926
927 video_codecs_.clear();
928
929 bool found = false;
930 for (size_t i = 0; i < ARRAY_SIZE(kVideoCodecPrefs); ++i) {
931 const VideoCodecPref& pref(kVideoCodecPrefs[i]);
932 if (!found)
933 found = (in_codec.name == pref.name);
934 if (found) {
935 VideoCodec codec(pref.payload_type, pref.name,
936 in_codec.width, in_codec.height, in_codec.framerate,
937 ARRAY_SIZE(kVideoCodecPrefs) - i);
938 video_codecs_.push_back(codec);
939 }
940 }
941 ASSERT(found);
942 return true;
943 }
944
SetCapturer(VideoCapturer * capturer,bool own_capturer)945 bool WebRtcVideoEngine::SetCapturer(VideoCapturer* capturer,
946 bool own_capturer) {
947 if (capturer == NULL) {
948 // Stop capturing before clearing the capturer.
949 if (SetCapture(false) != CR_SUCCESS) {
950 LOG(LS_WARNING) << "Camera failed to stop";
951 return false;
952 }
953 ClearCapturer();
954 return true;
955 }
956 // Hook up signals and install the supplied capturer.
957 SignalCaptureResult.repeat(capturer->SignalStartResult);
958 capturer->SignalFrameCaptured.connect(this,
959 &WebRtcVideoEngine::OnFrameCaptured);
960 ClearCapturer();
961 video_capturer_ = capturer;
962 owns_capturer_ = own_capturer;
963 // Possibly restart the capturer if it is supposed to be running.
964 CaptureResult result = UpdateCapturingState();
965 if (result != CR_SUCCESS && result != CR_PENDING) {
966 LOG(LS_WARNING) << "Camera failed to restart";
967 return false;
968 }
969 return true;
970 }
971
PerformanceAlarm(const unsigned int cpu_load)972 void WebRtcVideoEngine::PerformanceAlarm(const unsigned int cpu_load) {
973 LOG(LS_INFO) << "WebRtcVideoEngine::PerformanceAlarm";
974 }
975
976 // Ignore spammy trace messages, mostly from the stats API when we haven't
977 // gotten RTCP info yet from the remote side.
ShouldIgnoreTrace(const std::string & trace)978 bool WebRtcVideoEngine::ShouldIgnoreTrace(const std::string& trace) {
979 static const char* const kTracesToIgnore[] = {
980 NULL
981 };
982 for (const char* const* p = kTracesToIgnore; *p; ++p) {
983 if (trace.find(*p) == 0) {
984 return true;
985 }
986 }
987 return false;
988 }
989
GetNumOfChannels()990 int WebRtcVideoEngine::GetNumOfChannels() {
991 talk_base::CritScope cs(&channels_crit_);
992 return channels_.size();
993 }
994
Print(const webrtc::TraceLevel level,const char * trace,const int length)995 void WebRtcVideoEngine::Print(const webrtc::TraceLevel level,
996 const char* trace, const int length) {
997 talk_base::LoggingSeverity sev = talk_base::LS_VERBOSE;
998 if (level == webrtc::kTraceError || level == webrtc::kTraceCritical)
999 sev = talk_base::LS_ERROR;
1000 else if (level == webrtc::kTraceWarning)
1001 sev = talk_base::LS_WARNING;
1002 else if (level == webrtc::kTraceStateInfo || level == webrtc::kTraceInfo)
1003 sev = talk_base::LS_INFO;
1004
1005 if (sev >= log_level_) {
1006 // Skip past boilerplate prefix text
1007 if (length < 72) {
1008 std::string msg(trace, length);
1009 LOG(LS_ERROR) << "Malformed webrtc log message: ";
1010 LOG_V(sev) << msg;
1011 } else {
1012 std::string msg(trace + 71, length - 72);
1013 if (!ShouldIgnoreTrace(msg) &&
1014 (!voice_engine_ || !voice_engine_->ShouldIgnoreTrace(msg))) {
1015 LOG_V(sev) << "WebRtc:" << msg;
1016 }
1017 }
1018 }
1019 }
1020
RegisterProcessor(VideoProcessor * video_processor)1021 bool WebRtcVideoEngine::RegisterProcessor(
1022 VideoProcessor* video_processor) {
1023 talk_base::CritScope cs(&signal_media_critical_);
1024 SignalMediaFrame.connect(video_processor,
1025 &VideoProcessor::OnFrame);
1026 return true;
1027 }
UnregisterProcessor(VideoProcessor * video_processor)1028 bool WebRtcVideoEngine::UnregisterProcessor(
1029 VideoProcessor* video_processor) {
1030 talk_base::CritScope cs(&signal_media_critical_);
1031 SignalMediaFrame.disconnect(video_processor);
1032 return true;
1033 }
1034
ClearCapturer()1035 void WebRtcVideoEngine::ClearCapturer() {
1036 if (owns_capturer_) {
1037 delete video_capturer_;
1038 }
1039 video_capturer_ = NULL;
1040 }
1041
1042 // WebRtcVideoMediaChannel
1043
WebRtcVideoMediaChannel(WebRtcVideoEngine * engine,VoiceMediaChannel * channel)1044 WebRtcVideoMediaChannel::WebRtcVideoMediaChannel(
1045 WebRtcVideoEngine* engine, VoiceMediaChannel* channel)
1046 : engine_(engine),
1047 voice_channel_(channel),
1048 vie_channel_(-1),
1049 vie_capture_(-1),
1050 external_capture_(NULL),
1051 render_started_(false),
1052 muted_(false),
1053 first_receive_ssrc_(0),
1054 send_min_bitrate_(kMinVideoBitrate),
1055 send_start_bitrate_(kStartVideoBitrate),
1056 send_max_bitrate_(kMaxVideoBitrate),
1057 sending_(false),
1058 local_stream_info_(new WebRtcLocalStreamInfo()),
1059 options_(0) {
1060 engine->RegisterChannel(this);
1061 }
1062
Init()1063 bool WebRtcVideoMediaChannel::Init() {
1064 if (engine_->vie()->base()->CreateChannel(vie_channel_) != 0) {
1065 LOG_RTCERR1(CreateChannel, vie_channel_);
1066 return false;
1067 }
1068 if (!ConfigureChannel(vie_channel_)) {
1069 engine_->vie()->base()->DeleteChannel(vie_channel_);
1070 vie_channel_ = -1;
1071 return false;
1072 }
1073
1074 if (!ConfigureReceiving(vie_channel_, 0)) {
1075 engine_->vie()->base()->DeleteChannel(vie_channel_);
1076 vie_channel_ = -1;
1077 return false;
1078 }
1079
1080 LOG(LS_INFO) << "WebRtcVideoMediaChannel::Init "
1081 << "vie_channel " << vie_channel_ << " created";
1082
1083 // Register external capture.
1084 if (engine()->vie()->capture()->AllocateExternalCaptureDevice(
1085 vie_capture_, external_capture_) != 0) {
1086 LOG_RTCERR0(AllocateExternalCaptureDevice);
1087 return false;
1088 }
1089
1090 // Connect external capture.
1091 if (engine()->vie()->capture()->ConnectCaptureDevice(
1092 vie_capture_, vie_channel_) != 0) {
1093 LOG_RTCERR2(ConnectCaptureDevice, vie_capture_, vie_channel_);
1094 return false;
1095 }
1096
1097 // Register encoder observer for outgoing framerate and bitrate.
1098 encoder_observer_.reset(new WebRtcEncoderObserver(vie_channel_));
1099 if (engine()->vie()->codec()->RegisterEncoderObserver(
1100 vie_channel_, *encoder_observer_) != 0) {
1101 LOG_RTCERR1(RegisterEncoderObserver, encoder_observer_.get());
1102 return false;
1103 }
1104
1105 return true;
1106 }
1107
~WebRtcVideoMediaChannel()1108 WebRtcVideoMediaChannel::~WebRtcVideoMediaChannel() {
1109 if (vie_channel_ != -1) {
1110 // Stop sending.
1111 SetSend(false);
1112 if (engine()->vie()->codec()->DeregisterEncoderObserver(
1113 vie_channel_) != 0) {
1114 LOG_RTCERR1(DeregisterEncoderObserver, vie_channel_);
1115 }
1116
1117 // Stop the renderer.
1118 SetRender(false);
1119
1120 // Destroy the external capture interface.
1121 if (vie_capture_ != -1) {
1122 if (engine()->vie()->capture()->DisconnectCaptureDevice(
1123 vie_channel_) != 0) {
1124 LOG_RTCERR1(DisconnectCaptureDevice, vie_channel_);
1125 }
1126 if (engine()->vie()->capture()->ReleaseCaptureDevice(
1127 vie_capture_) != 0) {
1128 LOG_RTCERR1(ReleaseCaptureDevice, vie_capture_);
1129 }
1130 }
1131
1132 // Remove all receive streams and the default channel.
1133 while (!mux_channels_.empty()) {
1134 RemoveRecvStream(mux_channels_.begin()->first);
1135 }
1136 }
1137
1138 // Unregister the channel from the engine.
1139 engine()->UnregisterChannel(this);
1140 }
1141
SetRecvCodecs(const std::vector<VideoCodec> & codecs)1142 bool WebRtcVideoMediaChannel::SetRecvCodecs(
1143 const std::vector<VideoCodec>& codecs) {
1144 receive_codecs_.clear();
1145 for (std::vector<VideoCodec>::const_iterator iter = codecs.begin();
1146 iter != codecs.end(); ++iter) {
1147 if (engine()->FindCodec(*iter)) {
1148 webrtc::VideoCodec wcodec;
1149 if (engine()->ConvertFromCricketVideoCodec(*iter, &wcodec)) {
1150 receive_codecs_.push_back(wcodec);
1151 }
1152 } else {
1153 LOG(LS_INFO) << "Unknown codec " << iter->name;
1154 return false;
1155 }
1156 }
1157
1158 for (ChannelMap::iterator it = mux_channels_.begin();
1159 it != mux_channels_.end(); ++it) {
1160 if (!SetReceiveCodecs(it->second->channel_id()))
1161 return false;
1162 }
1163 return true;
1164 }
1165
SetSendCodecs(const std::vector<VideoCodec> & codecs)1166 bool WebRtcVideoMediaChannel::SetSendCodecs(
1167 const std::vector<VideoCodec>& codecs) {
1168 // Match with local video codec list.
1169 std::vector<webrtc::VideoCodec> send_codecs;
1170 int red_type = -1, fec_type = -1;
1171 VideoCodec checked_codec;
1172 VideoCodec current; // defaults to 0x0
1173 if (sending_) {
1174 engine()->ConvertToCricketVideoCodec(*send_codec_, ¤t);
1175 }
1176 for (std::vector<VideoCodec>::const_iterator iter = codecs.begin();
1177 iter != codecs.end(); ++iter) {
1178 if (_stricmp(iter->name.c_str(), kRedPayloadName) == 0) {
1179 red_type = iter->id;
1180 } else if (_stricmp(iter->name.c_str(), kFecPayloadName) == 0) {
1181 fec_type = iter->id;
1182 } else if (engine()->CanSendCodec(*iter, current, &checked_codec)) {
1183 webrtc::VideoCodec wcodec;
1184 if (engine()->ConvertFromCricketVideoCodec(checked_codec, &wcodec)) {
1185 send_codecs.push_back(wcodec);
1186 }
1187 } else {
1188 LOG(LS_WARNING) << "Unknown codec " << iter->name;
1189 }
1190 }
1191
1192 // Fail if we don't have a match.
1193 if (send_codecs.empty()) {
1194 LOG(LS_WARNING) << "No matching codecs avilable";
1195 return false;
1196 }
1197
1198 // Configure video protection.
1199 if (!SetNackFec(vie_channel_, red_type, fec_type)) {
1200 return false;
1201 }
1202
1203 // Select the first matched codec.
1204 webrtc::VideoCodec& codec(send_codecs[0]);
1205
1206 // Set the default number of temporal layers for VP8.
1207 if (webrtc::kVideoCodecVP8 == codec.codecType) {
1208 codec.codecSpecific.VP8.numberOfTemporalLayers =
1209 kDefaultNumberOfTemporalLayers;
1210 // Turn off the VP8 error resilience
1211 codec.codecSpecific.VP8.resilience = webrtc::kResilienceOff;
1212 }
1213
1214 if (!SetSendCodec(
1215 codec, send_min_bitrate_, send_start_bitrate_, send_max_bitrate_)) {
1216 return false;
1217 }
1218 LogSendCodecChange("SetSendCodecs()");
1219
1220 return true;
1221 }
1222
SetSendStreamFormat(uint32 ssrc,const VideoFormat & format)1223 bool WebRtcVideoMediaChannel::SetSendStreamFormat(uint32 ssrc,
1224 const VideoFormat& format) {
1225 if (send_params_.get() == NULL) {
1226 LOG(LS_WARNING) << "Sending stream has not been added yet.";
1227 return false;
1228 }
1229 if (!send_params_->has_ssrc(ssrc)) {
1230 LOG(LS_ERROR) << "The specified ssrc " << ssrc << " is not in use.";
1231 return false;
1232 }
1233
1234 if (send_codec_.get() == NULL) {
1235 LOG(LS_ERROR) << "The send codec has not been set yet.";
1236 return false;
1237 }
1238
1239 webrtc::VideoCodec codec = *send_codec_.get();
1240 codec.width = format.width;
1241 codec.height = format.height;
1242 codec.maxFramerate = VideoFormat::IntervalToFps(format.interval);
1243
1244 bool ret = SetSendCodec(
1245 codec, send_min_bitrate_, send_start_bitrate_, send_max_bitrate_);
1246 if (ret) {
1247 LogSendCodecChange("SetSendStreamFormat()");
1248 }
1249 return ret;
1250 }
1251
SetRender(bool render)1252 bool WebRtcVideoMediaChannel::SetRender(bool render) {
1253 if (render == render_started_) {
1254 return true; // no action required
1255 }
1256
1257 bool ret = true;
1258 for (ChannelMap::iterator it = mux_channels_.begin();
1259 it != mux_channels_.end(); ++it) {
1260 if (render) {
1261 if (engine()->vie()->render()->StartRender(
1262 it->second->channel_id()) != 0) {
1263 LOG_RTCERR1(StartRender, it->second->channel_id());
1264 ret = false;
1265 }
1266 } else {
1267 if (engine()->vie()->render()->StopRender(
1268 it->second->channel_id()) != 0) {
1269 LOG_RTCERR1(StopRender, it->second->channel_id());
1270 ret = false;
1271 }
1272 }
1273 }
1274 if (ret) {
1275 render_started_ = render;
1276 }
1277
1278 return ret;
1279 }
1280
SetSend(bool send)1281 bool WebRtcVideoMediaChannel::SetSend(bool send) {
1282 if (send_params_.get() == NULL && send) {
1283 LOG(LS_ERROR) << "No stream added";
1284 return false;
1285 }
1286 if (send == sending()) {
1287 return true; // No action required.
1288 }
1289
1290 if (send) {
1291 // We've been asked to start sending.
1292 // SetSendCodecs must have been called already.
1293 if (!send_codec_.get()) {
1294 return false;
1295 }
1296 // Start send now.
1297 if (!StartSend()) {
1298 return false;
1299 }
1300 } else {
1301 // We've been asked to stop sending.
1302 if (!StopSend()) {
1303 return false;
1304 }
1305 }
1306 sending_ = send;
1307
1308 return true;
1309 }
1310
GetChannelNum(uint32 ssrc)1311 int WebRtcVideoMediaChannel::GetChannelNum(uint32 ssrc) {
1312 ChannelMap::iterator it = mux_channels_.find(ssrc);
1313 return (it != mux_channels_.end()) ? it->second->channel_id() : -1;
1314 }
1315
AddSendStream(const StreamParams & sp)1316 bool WebRtcVideoMediaChannel::AddSendStream(const StreamParams& sp) {
1317 if (send_params_.get() != NULL) {
1318 LOG(LS_ERROR) << "WebRtcVideoMediaChannel supports one sending channel";
1319 return false;
1320 }
1321
1322 if (!IsOneSsrcStream(sp)) {
1323 LOG(LS_ERROR) << "AddSendStream: bad local stream parameters";
1324 return false;
1325 }
1326
1327 // Set the send (local) SSRC.
1328 // If there are multiple send SSRCs, we can only set the first one here, and
1329 // the rest of the SSRC(s) need to be set after SetSendCodec has been called
1330 // (with a codec requires multiple SSRC(s)).
1331 if (engine()->vie()->rtp()->SetLocalSSRC(vie_channel_,
1332 sp.first_ssrc()) != 0) {
1333 LOG_RTCERR2(SetLocalSSRC, vie_channel_, sp.first_ssrc());
1334 return false;
1335 }
1336
1337 // Set RTCP CName.
1338 if (engine()->vie()->rtp()->SetRTCPCName(vie_channel_,
1339 sp.cname.c_str()) != 0) {
1340 LOG_RTCERR2(SetRTCPCName, vie_channel_, sp.cname.c_str());
1341 return false;
1342 }
1343
1344 // Set the SSRC on the receive channels and this send channel.
1345 // Receive channels have to have the same SSRC in order to send receiver
1346 // reports with this SSRC.
1347 for (ChannelMap::const_iterator it = mux_channels_.begin();
1348 it != mux_channels_.end(); ++it) {
1349 WebRtcVideoChannelInfo* info = it->second;
1350 int channel_id = info->channel_id();
1351 if (engine()->vie()->rtp()->SetLocalSSRC(channel_id,
1352 sp.first_ssrc()) != 0) {
1353 LOG_RTCERR1(SetLocalSSRC, it->first);
1354 return false;
1355 }
1356 }
1357
1358 // Save the StreamParams.
1359 send_params_.reset(new StreamParams(sp));
1360
1361 // Reset send codec after stream parameters changed.
1362 if (send_codec_.get() != NULL) {
1363 if (!SetSendCodec(*send_codec_, send_min_bitrate_,
1364 send_start_bitrate_, send_max_bitrate_)) {
1365 return false;
1366 }
1367 LogSendCodecChange("SetSendStreamFormat()");
1368 }
1369
1370 if (sending_) {
1371 return StartSend();
1372 }
1373 return true;
1374 }
1375
RemoveSendStream(uint32 ssrc)1376 bool WebRtcVideoMediaChannel::RemoveSendStream(uint32 ssrc) {
1377 if (send_params_.get() == NULL || !send_params_->has_ssrc(ssrc)) {
1378 LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc
1379 << " which doesn't exist.";
1380 return false;
1381 }
1382 if (sending_) {
1383 StopSend();
1384 }
1385 send_params_.reset(NULL);
1386 return true;
1387 }
1388
AddRecvStream(const StreamParams & sp)1389 bool WebRtcVideoMediaChannel::AddRecvStream(const StreamParams& sp) {
1390 // TODO Remove this once BWE works properly across different send
1391 // and receive channels.
1392 // Reuse default channel for recv stream in 1:1 call.
1393 if ((options_ & OPT_CONFERENCE) == 0 && first_receive_ssrc_ == 0) {
1394 LOG(LS_INFO) << "Recv stream " << sp.first_ssrc()
1395 << " reuse default channel #"
1396 << vie_channel_;
1397 first_receive_ssrc_ = sp.first_ssrc();
1398 return true;
1399 }
1400
1401 if (mux_channels_.find(sp.first_ssrc()) != mux_channels_.end()) {
1402 LOG(LS_ERROR) << "Stream already exists";
1403 return false;
1404 }
1405
1406 // TODO: Implement recv media from multiple SSRCs per stream.
1407 if (sp.ssrcs.size() != 1) {
1408 LOG(LS_ERROR) << "WebRtcVideoMediaChannel supports one receiving SSRC per"
1409 << " stream";
1410 return false;
1411 }
1412
1413 // Create a new channel for receiving video data.
1414 // In order to get the bandwidth estimation work fine for
1415 // receive only channels, we connect all receiving channels
1416 // to our master send channel.
1417 int channel_id = -1;
1418 if (engine_->vie()->base()->CreateReceiveChannel(channel_id,
1419 vie_channel_) != 0) {
1420 LOG_RTCERR2(CreateReceiveChannel, channel_id, vie_channel_);
1421 return false;
1422 }
1423
1424 // Get the default renderer.
1425 VideoRenderer* default_renderer = NULL;
1426 if ((options_ & OPT_CONFERENCE) != 0) {
1427 if (mux_channels_.size() == 1 &&
1428 mux_channels_.find(0) != mux_channels_.end()) {
1429 GetRenderer(0, &default_renderer);
1430 }
1431 }
1432
1433 if (!ConfigureChannel(channel_id) ||
1434 !ConfigureReceiving(channel_id, sp.first_ssrc())) {
1435 engine_->vie()->base()->DeleteChannel(channel_id);
1436 return false;
1437 }
1438
1439 // The first recv stream reuses the default renderer (if a default renderer
1440 // has been set).
1441 if (default_renderer) {
1442 SetRenderer(sp.first_ssrc(), default_renderer);
1443 }
1444
1445 LOG(LS_INFO) << "New video stream " << sp.first_ssrc()
1446 << " registered to VideoEngine channel #"
1447 << channel_id << " and connected to channel #" << vie_channel_;
1448
1449 return true;
1450 }
1451
RemoveRecvStream(uint32 ssrc)1452 bool WebRtcVideoMediaChannel::RemoveRecvStream(uint32 ssrc) {
1453 ChannelMap::iterator it = mux_channels_.find(ssrc);
1454
1455 if (it == mux_channels_.end()) {
1456 // TODO: Remove this once BWE works properly across different send
1457 // and receive channels.
1458 // The default channel is reused for recv stream in 1:1 call.
1459 if (first_receive_ssrc_ == ssrc) {
1460 first_receive_ssrc_ = 0;
1461 return true;
1462 }
1463 return false;
1464 }
1465 WebRtcVideoChannelInfo* info = it->second;
1466 int channel_id = info->channel_id();
1467 if (engine()->vie()->render()->RemoveRenderer(channel_id) != 0) {
1468 LOG_RTCERR1(RemoveRenderer, channel_id);
1469 }
1470
1471 if (engine()->vie()->network()->DeregisterSendTransport(channel_id) !=0) {
1472 LOG_RTCERR1(DeRegisterSendTransport, channel_id);
1473 }
1474
1475 if (engine()->vie()->codec()->DeregisterDecoderObserver(
1476 channel_id) != 0) {
1477 LOG_RTCERR1(DeregisterDecoderObserver, channel_id);
1478 }
1479
1480 LOG(LS_INFO) << "Removing video stream " << ssrc
1481 << " with VideoEngine channel #"
1482 << channel_id;
1483 if (engine()->vie()->base()->DeleteChannel(channel_id) == -1) {
1484 LOG_RTCERR1(DeleteChannel, channel_id);
1485 // Leak the WebRtcVideoChannelInfo owned by |it| but remove the channel from
1486 // mux_channels_.
1487 mux_channels_.erase(it);
1488 return false;
1489 }
1490 // Delete the WebRtcVideoChannelInfo pointed to by it->second.
1491 delete info;
1492 mux_channels_.erase(it);
1493 return true;
1494 }
1495
StartSend()1496 bool WebRtcVideoMediaChannel::StartSend() {
1497 if (engine()->vie()->base()->StartSend(vie_channel_) != 0) {
1498 LOG_RTCERR1(StartSend, vie_channel_);
1499 return false;
1500 }
1501
1502 // TODO Change this once REMB supporting multiple sending channels.
1503 // Send remb (2nd param) and use remb for BWE (3rd param).
1504 if (engine_->vie()->rtp()->SetRembStatus(vie_channel_, true, true) != 0) {
1505 LOG_RTCERR3(SetRembStatus, vie_channel_, true, true);
1506 return false;
1507 }
1508
1509 return true;
1510 }
1511
StopSend()1512 bool WebRtcVideoMediaChannel::StopSend() {
1513 if (engine()->vie()->base()->StopSend(vie_channel_) != 0) {
1514 LOG_RTCERR1(StopSend, vie_channel_);
1515 return false;
1516 }
1517
1518 // TODO Change this once REMB supporting multiple sending channels.
1519 // Don't send remb (2nd param) but use remb for BWE (3rd param).
1520 if (engine_->vie()->rtp()->SetRembStatus(vie_channel_, false, true) != 0) {
1521 LOG_RTCERR3(SetRembStatus, vie_channel_, false, true);
1522 return false;
1523 }
1524
1525 return true;
1526 }
1527
IsOneSsrcStream(const StreamParams & sp)1528 bool WebRtcVideoMediaChannel::IsOneSsrcStream(const StreamParams& sp) {
1529 return (sp.ssrcs.size() == 1 && sp.ssrc_groups.size() == 0);
1530 }
1531
SetRenderer(uint32 ssrc,VideoRenderer * renderer)1532 bool WebRtcVideoMediaChannel::SetRenderer(uint32 ssrc,
1533 VideoRenderer* renderer) {
1534 if (mux_channels_.find(ssrc) == mux_channels_.end()) {
1535 // TODO: Remove this once BWE works properly across different send
1536 // and receive channels.
1537 // The default channel is reused for recv stream in 1:1 call.
1538 if (first_receive_ssrc_ == ssrc &&
1539 mux_channels_.find(0) != mux_channels_.end()) {
1540 LOG(LS_INFO) << "SetRenderer " << ssrc
1541 << " reuse default channel #"
1542 << vie_channel_;
1543 mux_channels_[0]->SetRenderer(renderer);
1544 return true;
1545 }
1546 return false;
1547 }
1548
1549 mux_channels_[ssrc]->SetRenderer(renderer);
1550 return true;
1551 }
1552
GetStats(VideoMediaInfo * info)1553 bool WebRtcVideoMediaChannel::GetStats(VideoMediaInfo* info) {
1554 // Get basic statistics.
1555 unsigned int bytes_sent, packets_sent, bytes_recv, packets_recv;
1556 unsigned int ssrc;
1557 if (engine_->vie()->rtp()->GetRTPStatistics(vie_channel_,
1558 bytes_sent, packets_sent, bytes_recv, packets_recv) != 0) {
1559 LOG_RTCERR1(GetRTPStatistics, vie_channel_);
1560 return false;
1561 }
1562
1563 // Get sender statistics and build VideoSenderInfo.
1564 if (send_params_.get() != NULL && send_codec_.get() != NULL) {
1565 VideoSenderInfo sinfo;
1566 sinfo.ssrcs = send_params_->ssrcs;
1567 sinfo.codec_name = send_codec_.get() ? send_codec_->plName : "";
1568 sinfo.bytes_sent = bytes_sent;
1569 sinfo.packets_sent = packets_sent;
1570 sinfo.packets_cached = -1;
1571 sinfo.packets_lost = -1;
1572 sinfo.fraction_lost = -1;
1573 sinfo.firs_rcvd = -1;
1574 sinfo.nacks_rcvd = -1;
1575 sinfo.rtt_ms = -1;
1576 sinfo.frame_width = local_stream_info_->width();
1577 sinfo.frame_height = local_stream_info_->height();
1578 sinfo.framerate_input = local_stream_info_->framerate();
1579 sinfo.framerate_sent = encoder_observer_->framerate();
1580 sinfo.nominal_bitrate = encoder_observer_->bitrate();
1581 sinfo.preferred_bitrate = send_max_bitrate_;
1582
1583 // Get received RTCP statistics for the sender, if available.
1584 // It's not a fatal error if we can't, since RTCP may not have arrived yet.
1585 uint16 r_fraction_lost;
1586 unsigned int r_cumulative_lost;
1587 unsigned int r_extended_max;
1588 unsigned int r_jitter;
1589 int r_rtt_ms;
1590 if (engine_->vie()->rtp()->GetSentRTCPStatistics(vie_channel_,
1591 r_fraction_lost, r_cumulative_lost, r_extended_max,
1592 r_jitter, r_rtt_ms) == 0) {
1593 // Convert Q8 to float.
1594 sinfo.packets_lost = r_cumulative_lost;
1595 sinfo.fraction_lost = static_cast<float>(r_fraction_lost) / (1 << 8);
1596 sinfo.rtt_ms = r_rtt_ms;
1597 }
1598 info->senders.push_back(sinfo);
1599 } else {
1600 LOG(LS_WARNING) << "GetStats: sender information not ready.";
1601 }
1602
1603 // Get the SSRC and stats for each receiver, based on our own calculations.
1604 for (ChannelMap::const_iterator it = mux_channels_.begin();
1605 it != mux_channels_.end(); ++it) {
1606 // Don't report receive statistics from the default channel if we have
1607 // specified receive channels.
1608 if (it->first == 0 && mux_channels_.size() > 1)
1609 continue;
1610 WebRtcVideoChannelInfo* channel = it->second;
1611
1612 // Get receiver statistics and build VideoReceiverInfo, if we have data.
1613 if (engine_->vie()->rtp()->GetRemoteSSRC(channel->channel_id(), ssrc) != 0)
1614 continue;
1615
1616 if (engine_->vie()->rtp()->GetRTPStatistics(
1617 channel->channel_id(), bytes_sent, packets_sent, bytes_recv,
1618 packets_recv) != 0) {
1619 LOG_RTCERR1(GetRTPStatistics, channel->channel_id());
1620 return false;
1621 }
1622 VideoReceiverInfo rinfo;
1623 rinfo.ssrcs.push_back(ssrc);
1624 rinfo.bytes_rcvd = bytes_recv;
1625 rinfo.packets_rcvd = packets_recv;
1626 rinfo.packets_lost = -1;
1627 rinfo.packets_concealed = -1;
1628 rinfo.fraction_lost = -1; // from SentRTCP
1629 rinfo.firs_sent = channel->decoder_observer()->firs_requested();
1630 rinfo.nacks_sent = -1;
1631 rinfo.frame_width = channel->render_adapter()->width();
1632 rinfo.frame_height = channel->render_adapter()->height();
1633 rinfo.framerate_rcvd = channel->decoder_observer()->framerate();
1634 int fps = channel->render_adapter()->framerate();
1635 rinfo.framerate_decoded = fps;
1636 rinfo.framerate_output = fps;
1637
1638 // Get sent RTCP statistics.
1639 uint16 s_fraction_lost;
1640 unsigned int s_cumulative_lost;
1641 unsigned int s_extended_max;
1642 unsigned int s_jitter;
1643 int s_rtt_ms;
1644 if (engine_->vie()->rtp()->GetReceivedRTCPStatistics(channel->channel_id(),
1645 s_fraction_lost, s_cumulative_lost, s_extended_max,
1646 s_jitter, s_rtt_ms) == 0) {
1647 // Convert Q8 to float.
1648 rinfo.packets_lost = s_cumulative_lost;
1649 rinfo.fraction_lost = static_cast<float>(s_fraction_lost) / (1 << 8);
1650 }
1651 info->receivers.push_back(rinfo);
1652 }
1653
1654 // Build BandwidthEstimationInfo.
1655 // TODO: Fill in more BWE stats once we have them.
1656 unsigned int total_bitrate_sent;
1657 unsigned int video_bitrate_sent;
1658 unsigned int fec_bitrate_sent;
1659 unsigned int nack_bitrate_sent;
1660 if (engine_->vie()->rtp()->GetBandwidthUsage(vie_channel_,
1661 total_bitrate_sent, video_bitrate_sent,
1662 fec_bitrate_sent, nack_bitrate_sent) == 0) {
1663 BandwidthEstimationInfo bwe;
1664 bwe.actual_enc_bitrate = video_bitrate_sent;
1665 bwe.transmit_bitrate = total_bitrate_sent;
1666 bwe.retransmit_bitrate = nack_bitrate_sent;
1667
1668 // Add bandwidth estimation.
1669 unsigned int estimated_send_bandwidth;
1670 if (engine_->vie()->rtp()->GetEstimatedSendBandwidth(
1671 vie_channel_, &estimated_send_bandwidth)) {
1672 bwe.available_send_bandwidth = estimated_send_bandwidth;
1673 } else {
1674 LOG_RTCERR2(GetEstimatedSendBandwidth, vie_channel_,
1675 &estimated_send_bandwidth);
1676 }
1677 unsigned int estimated_recv_bandwidth;
1678 if (engine_->vie()->rtp()->GetEstimatedReceiveBandwidth(
1679 vie_channel_, &estimated_recv_bandwidth)) {
1680 bwe.available_recv_bandwidth = estimated_recv_bandwidth;
1681 } else {
1682 LOG_RTCERR2(GetEstimatedRecvBandwidth, vie_channel_,
1683 &estimated_recv_bandwidth);
1684 }
1685 unsigned int target_enc_bitrate;
1686 if (engine_->vie()->codec()->GetCodecTargetBitrate(
1687 vie_channel_, &target_enc_bitrate)) {
1688 bwe.target_enc_bitrate = target_enc_bitrate;
1689 } else {
1690 LOG_RTCERR2(GetCodecTargetBitrate, vie_channel_,
1691 &target_enc_bitrate);
1692 }
1693
1694 info->bw_estimations.push_back(bwe);
1695 } else {
1696 LOG_RTCERR1(GetBandwidthUsage, vie_channel_);
1697 }
1698
1699 return true;
1700 }
1701
SendIntraFrame()1702 bool WebRtcVideoMediaChannel::SendIntraFrame() {
1703 bool ret = true;
1704 if (engine()->vie()->codec()->SendKeyFrame(vie_channel_) != 0) {
1705 LOG_RTCERR1(SendKeyFrame, vie_channel_);
1706 ret = false;
1707 }
1708
1709 return ret;
1710 }
1711
RequestIntraFrame()1712 bool WebRtcVideoMediaChannel::RequestIntraFrame() {
1713 // There is no API exposed to application to request a key frame
1714 // ViE does this internally when there are errors from decoder
1715 return false;
1716 }
1717
OnPacketReceived(talk_base::Buffer * packet)1718 void WebRtcVideoMediaChannel::OnPacketReceived(talk_base::Buffer* packet) {
1719 // Pick which channel to send this packet to. If this packet doesn't match
1720 // any multiplexed streams, just send it to the default channel. Otherwise,
1721 // send it to the specific decoder instance for that stream.
1722 uint32 ssrc = 0;
1723 if (!GetRtpSsrc(packet->data(), packet->length(), &ssrc))
1724 return;
1725 int which_channel = GetChannelNum(ssrc);
1726 if (which_channel == -1) {
1727 which_channel = video_channel();
1728 }
1729
1730 engine()->vie()->network()->ReceivedRTPPacket(which_channel,
1731 packet->data(),
1732 packet->length());
1733 }
1734
OnRtcpReceived(talk_base::Buffer * packet)1735 void WebRtcVideoMediaChannel::OnRtcpReceived(talk_base::Buffer* packet) {
1736 // Sending channels need all RTCP packets with feedback information.
1737 // Even sender reports can contain attached report blocks.
1738 // Receiving channels need sender reports in order to create
1739 // correct receiver reports.
1740
1741 uint32 ssrc = 0;
1742 if (!GetRtcpSsrc(packet->data(), packet->length(), &ssrc)) {
1743 LOG(LS_WARNING) << "Failed to parse SSRC from received RTCP packet";
1744 return;
1745 }
1746 int type = 0;
1747 if (!GetRtcpType(packet->data(), packet->length(), & type)) {
1748 LOG(LS_WARNING) << "Failed to parse type from received RTCP packet";
1749 return;
1750 }
1751
1752 // If it is a sender report, find the channel that is listening.
1753 if (type == kRtcpTypeSR) {
1754 int which_channel = GetChannelNum(ssrc);
1755 if (which_channel != -1 && which_channel != vie_channel_) {
1756 engine_->vie()->network()->ReceivedRTCPPacket(which_channel,
1757 packet->data(),
1758 packet->length());
1759 }
1760 }
1761 // The sending channel receives all RTCP packets.
1762 engine_->vie()->network()->ReceivedRTCPPacket(vie_channel_,
1763 packet->data(),
1764 packet->length());
1765 }
1766
Mute(bool on)1767 bool WebRtcVideoMediaChannel::Mute(bool on) {
1768 muted_ = on;
1769 return true;
1770 }
1771
SetSendBandwidth(bool autobw,int bps)1772 bool WebRtcVideoMediaChannel::SetSendBandwidth(bool autobw, int bps) {
1773 LOG(LS_INFO) << "WebRtcVideoMediaChanne::SetSendBandwidth";
1774
1775 if (0 != (options_ & OPT_CONFERENCE)) {
1776 LOG(LS_INFO) << "Conference mode ignores SetSendBandWidth";
1777 return true;
1778 }
1779
1780 if (!send_codec_.get()) {
1781 LOG(LS_INFO) << "The send codec has not been set up yet";
1782 return true;
1783 }
1784
1785 int min_bitrate;
1786 int start_bitrate;
1787 int max_bitrate;
1788 if (autobw) {
1789 // Use the default values for min bitrate.
1790 min_bitrate = kMinVideoBitrate;
1791 // Use the default value or the bps for the max
1792 max_bitrate = (bps <= 0) ? send_max_bitrate_ : (bps / 1000);
1793 // Maximum start bitrate can be kStartVideoBitrate.
1794 start_bitrate = talk_base::_min(kStartVideoBitrate, max_bitrate);
1795 } else {
1796 // Use the default start or the bps as the target bitrate.
1797 int target_bitrate = (bps <= 0) ? kStartVideoBitrate : (bps / 1000);
1798 min_bitrate = target_bitrate;
1799 start_bitrate = target_bitrate;
1800 max_bitrate = target_bitrate;
1801 }
1802
1803 if (!SetSendCodec(*send_codec_, min_bitrate, start_bitrate, max_bitrate)) {
1804 return false;
1805 }
1806 LogSendCodecChange("SetSendBandwidth()");
1807
1808 return true;
1809 }
1810
SetOptions(int options)1811 bool WebRtcVideoMediaChannel::SetOptions(int options) {
1812 // Always accept options that are unchanged.
1813 if (options_ == options) {
1814 return true;
1815 }
1816
1817 // Reject new options if we're already sending.
1818 if (sending()) {
1819 return false;
1820 }
1821
1822 // Save the options, to be interpreted where appropriate.
1823 options_ = options;
1824
1825 // Adjust send codec bitrate if needed.
1826 int conf_max_bitrate = kDefaultConferenceModeMaxVideoBitrate;
1827 int expected_bitrate = (0 != (options_ & OPT_CONFERENCE)) ?
1828 conf_max_bitrate : kMaxVideoBitrate;
1829 if (NULL != send_codec_.get() && send_max_bitrate_ != expected_bitrate) {
1830 // On success, SetSendCodec() will reset send_max_bitrate_ to
1831 // expected_bitrate.
1832 if (!SetSendCodec(*send_codec_,
1833 send_min_bitrate_,
1834 send_start_bitrate_,
1835 expected_bitrate)) {
1836 return false;
1837 }
1838 LogSendCodecChange("SetOptions()");
1839 }
1840
1841 // Enable denoising if needed.
1842 if (vie_capture_ != -1) {
1843 bool enable = (options_ & OPT_VIDEO_NOISE_REDUCTION) != 0;
1844 // The EnableDenoising may return -1 when the denoising is already
1845 // enabled/disabled, which should not be treated as an error.
1846 // TODO: Return false once EnableDenoising only
1847 // reports the real failure.
1848 engine()->vie()->image()->EnableDenoising(vie_capture_, enable);
1849 } else {
1850 LOG(LS_WARNING) << "SetOptions: Video Capture is not ready.";
1851 }
1852
1853 return true;
1854 }
1855
SetInterface(NetworkInterface * iface)1856 void WebRtcVideoMediaChannel::SetInterface(NetworkInterface* iface) {
1857 MediaChannel::SetInterface(iface);
1858 // Set the RTP recv/send buffer to a bigger size
1859 if (network_interface_) {
1860 network_interface_->SetOption(NetworkInterface::ST_RTP,
1861 talk_base::Socket::OPT_RCVBUF,
1862 kVideoRtpBufferSize);
1863 network_interface_->SetOption(NetworkInterface::ST_RTP,
1864 talk_base::Socket::OPT_SNDBUF,
1865 kVideoRtpBufferSize);
1866 }
1867 }
1868
GetRenderer(uint32 ssrc,VideoRenderer ** renderer)1869 bool WebRtcVideoMediaChannel::GetRenderer(uint32 ssrc,
1870 VideoRenderer** renderer) {
1871 ChannelMap::const_iterator it = mux_channels_.find(ssrc);
1872 if (it == mux_channels_.end()) {
1873 if (first_receive_ssrc_ == ssrc &&
1874 mux_channels_.find(0) != mux_channels_.end()) {
1875 LOG(LS_INFO) << " GetRenderer " << ssrc
1876 << " reuse default renderer #"
1877 << vie_channel_;
1878 *renderer = mux_channels_[0]->render_adapter()->renderer();
1879 return true;
1880 }
1881 return false;
1882 }
1883
1884 *renderer = it->second->render_adapter()->renderer();
1885 return true;
1886 }
1887
1888 // TODO: Add unittests to test this function.
SendFrame(uint32 ssrc,const VideoFrame * frame)1889 bool WebRtcVideoMediaChannel::SendFrame(uint32 ssrc, const VideoFrame* frame) {
1890 if (ssrc != 0 || !sending() || !external_capture_) {
1891 return false;
1892 }
1893
1894 // Update local stream statistics.
1895 local_stream_info_->UpdateFrame(frame->GetWidth(), frame->GetHeight());
1896
1897 // Checks if we need to reset vie send codec.
1898 if (!MaybeResetVieSendCodec(frame->GetWidth(), frame->GetHeight(), NULL)) {
1899 LOG(LS_ERROR) << "MaybeResetVieSendCodec failed with "
1900 << frame->GetWidth() << "x" << frame->GetHeight();
1901 return false;
1902 }
1903
1904 // Blacken the frame if video is muted.
1905 const VideoFrame* frame_out = frame;
1906 talk_base::scoped_ptr<VideoFrame> black_frame;
1907 if (muted_) {
1908 black_frame.reset(frame->Copy());
1909 black_frame->SetToBlack();
1910 frame_out = black_frame.get();
1911 }
1912
1913 webrtc::ViEVideoFrameI420 frame_i420;
1914 // TODO: Update the webrtc::ViEVideoFrameI420
1915 // to use const unsigned char*
1916 frame_i420.y_plane = const_cast<unsigned char*>(frame_out->GetYPlane());
1917 frame_i420.u_plane = const_cast<unsigned char*>(frame_out->GetUPlane());
1918 frame_i420.v_plane = const_cast<unsigned char*>(frame_out->GetVPlane());
1919 frame_i420.y_pitch = frame_out->GetYPitch();
1920 frame_i420.u_pitch = frame_out->GetUPitch();
1921 frame_i420.v_pitch = frame_out->GetVPitch();
1922 frame_i420.width = frame_out->GetWidth();
1923 frame_i420.height = frame_out->GetHeight();
1924
1925 // Convert from nanoseconds to milliseconds.
1926 WebRtc_Word64 clocks = frame_out->GetTimeStamp() /
1927 talk_base::kNumNanosecsPerMillisec;
1928
1929 return (external_capture_->IncomingFrameI420(frame_i420, clocks) == 0);
1930 }
1931
ConfigureChannel(int channel_id)1932 bool WebRtcVideoMediaChannel::ConfigureChannel(int channel_id) {
1933 // Register external transport.
1934 if (engine_->vie()->network()->RegisterSendTransport(
1935 channel_id, *this) != 0) {
1936 LOG_RTCERR1(RegisterSendTransport, channel_id);
1937 return false;
1938 }
1939
1940 // Set MTU.
1941 if (engine_->vie()->network()->SetMTU(channel_id, kVideoMtu) != 0) {
1942 LOG_RTCERR2(SetMTU, channel_id, kVideoMtu);
1943 return false;
1944 }
1945 // Turn on RTCP and loss feedback reporting.
1946 if (engine()->vie()->rtp()->SetRTCPStatus(
1947 channel_id, webrtc::kRtcpCompound_RFC4585) != 0) {
1948 LOG_RTCERR2(SetRTCPStatus, channel_id, webrtc::kRtcpCompound_RFC4585);
1949 return false;
1950 }
1951 // Enable pli as key frame request method.
1952 if (engine_->vie()->rtp()->SetKeyFrameRequestMethod(
1953 channel_id, webrtc::kViEKeyFrameRequestPliRtcp) != 0) {
1954 LOG_RTCERR2(SetKeyFrameRequestMethod,
1955 channel_id, webrtc::kViEKeyFrameRequestPliRtcp);
1956 return false;
1957 }
1958 return true;
1959 }
1960
ConfigureReceiving(int channel_id,uint32 remote_ssrc)1961 bool WebRtcVideoMediaChannel::ConfigureReceiving(int channel_id,
1962 uint32 remote_ssrc) {
1963 // Connect the voice channel, if there is one.
1964 // TODO: The A/V is synched by the receiving channel. So we need to
1965 // know the SSRC of the remote audio channel in order to fetch the correct
1966 // webrtc VoiceEngine channel. For now- only sync the default channel used
1967 // in 1-1 calls.
1968 if (remote_ssrc == 0 && voice_channel_) {
1969 WebRtcVoiceMediaChannel* voice_channel =
1970 static_cast<WebRtcVoiceMediaChannel*>(voice_channel_);
1971 if (engine_->vie()->base()->ConnectAudioChannel(
1972 vie_channel_, voice_channel->voe_channel()) != 0) {
1973 LOG_RTCERR2(ConnectAudioChannel, channel_id,
1974 voice_channel->voe_channel());
1975 LOG(LS_WARNING) << "A/V not synchronized";
1976 // Not a fatal error.
1977 }
1978 }
1979
1980 talk_base::scoped_ptr<WebRtcVideoChannelInfo> channel_info(
1981 new WebRtcVideoChannelInfo(channel_id));
1982
1983 // Install a render adapter.
1984 if (engine_->vie()->render()->AddRenderer(channel_id,
1985 webrtc::kVideoI420, channel_info->render_adapter()) != 0) {
1986 LOG_RTCERR3(AddRenderer, channel_id, webrtc::kVideoI420,
1987 channel_info->render_adapter());
1988 return false;
1989 }
1990
1991 // TODO Change this once REMB supporting multiple sending channels.
1992 // Turn off remb sending (2nd param) and turn on remb reporting (3rd param)
1993 // here.
1994 // For sending channel, remb sending will be turned on after StartSending.
1995 if (engine_->vie()->rtp()->SetRembStatus(channel_id, false, true) != 0) {
1996 LOG_RTCERR3(SetRembStatus, vie_channel_, false, true);
1997 return false;
1998 }
1999
2000 if (remote_ssrc != 0) {
2001 // Use the same SSRC as our default channel
2002 // (so the RTCP reports are correct).
2003 unsigned int send_ssrc = 0;
2004 webrtc::ViERTP_RTCP* rtp = engine()->vie()->rtp();
2005 if (rtp->GetLocalSSRC(vie_channel_, send_ssrc) == -1) {
2006 LOG_RTCERR2(GetSendSSRC, channel_id, send_ssrc);
2007 return false;
2008 }
2009 if (rtp->SetLocalSSRC(channel_id, send_ssrc) == -1) {
2010 LOG_RTCERR2(SetSendSSRC, channel_id, send_ssrc);
2011 return false;
2012 }
2013 } // Else this is the the default channel and we don't change the SSRC.
2014
2015 // Disable color enhancement since it is a bit too aggressive.
2016 if (engine()->vie()->image()->EnableColorEnhancement(channel_id,
2017 false) != 0) {
2018 LOG_RTCERR1(EnableColorEnhancement, channel_id);
2019 return false;
2020 }
2021
2022 if (!SetReceiveCodecs(channel_id)) {
2023 return false;
2024 }
2025
2026 if (render_started_) {
2027 if (engine_->vie()->render()->StartRender(channel_id) != 0) {
2028 LOG_RTCERR1(StartRender, channel_id);
2029 return false;
2030 }
2031 }
2032
2033 // Register decoder observer for incoming framerate and bitrate.
2034 if (engine()->vie()->codec()->RegisterDecoderObserver(
2035 channel_id, *channel_info->decoder_observer()) != 0) {
2036 LOG_RTCERR1(RegisterDecoderObserver, channel_info->decoder_observer());
2037 return false;
2038 }
2039
2040 mux_channels_[remote_ssrc] = channel_info.release();
2041
2042 return true;
2043 }
2044
SetNackFec(int channel_id,int red_payload_type,int fec_payload_type)2045 bool WebRtcVideoMediaChannel::SetNackFec(int channel_id,
2046 int red_payload_type,
2047 int fec_payload_type) {
2048 // Enable hybrid NACK/FEC if negotiated and not in a conference, use only NACK
2049 // otherwise.
2050 bool enable = (red_payload_type != -1 && fec_payload_type != -1 &&
2051 !(options_ & OPT_CONFERENCE));
2052 if (enable) {
2053 if (engine_->vie()->rtp()->SetHybridNACKFECStatus(
2054 channel_id, enable, red_payload_type, fec_payload_type) != 0) {
2055 LOG_RTCERR4(SetHybridNACKFECStatus,
2056 channel_id, enable, red_payload_type, fec_payload_type);
2057 return false;
2058 }
2059 LOG(LS_INFO) << "Hybrid NACK/FEC enabled for channel " << channel_id;
2060 } else {
2061 if (engine_->vie()->rtp()->SetNACKStatus(channel_id, true) != 0) {
2062 LOG_RTCERR1(SetNACKStatus, channel_id);
2063 return false;
2064 }
2065 LOG(LS_INFO) << "NACK enabled for channel " << channel_id;
2066 }
2067 return true;
2068 }
2069
SetSendCodec(const webrtc::VideoCodec & codec,int min_bitrate,int start_bitrate,int max_bitrate)2070 bool WebRtcVideoMediaChannel::SetSendCodec(const webrtc::VideoCodec& codec,
2071 int min_bitrate,
2072 int start_bitrate,
2073 int max_bitrate) {
2074 // Make a copy of the codec
2075 webrtc::VideoCodec target_codec = codec;
2076 target_codec.startBitrate = start_bitrate;
2077 target_codec.minBitrate = min_bitrate;
2078 target_codec.maxBitrate = max_bitrate;
2079
2080 if (engine()->vie()->codec()->SetSendCodec(vie_channel_, target_codec) != 0) {
2081 LOG_RTCERR2(SetSendCodec, vie_channel_, send_codec_->plName);
2082 return false;
2083 }
2084
2085 // Reset the send_codec_ only if SetSendCodec is success.
2086 send_codec_.reset(new webrtc::VideoCodec(target_codec));
2087 send_min_bitrate_ = min_bitrate;
2088 send_start_bitrate_ = start_bitrate;
2089 send_max_bitrate_ = max_bitrate;
2090
2091 return true;
2092 }
2093
LogSendCodecChange(const std::string & reason)2094 void WebRtcVideoMediaChannel::LogSendCodecChange(const std::string& reason) {
2095 webrtc::VideoCodec vie_codec;
2096 if (engine()->vie()->codec()->GetSendCodec(vie_channel_, vie_codec) != 0) {
2097 LOG_RTCERR1(GetSendCodec, vie_channel_);
2098 return;
2099 }
2100
2101 LOG(LS_INFO) << reason << " : selected video codec "
2102 << vie_codec.plName << "/"
2103 << vie_codec.width << "x" << vie_codec.height << "x"
2104 << static_cast<int>(vie_codec.maxFramerate) << "fps"
2105 << "@" << vie_codec.maxBitrate << "kbps";
2106 if (webrtc::kVideoCodecVP8 == vie_codec.codecType) {
2107 LOG(LS_INFO) << "VP8 number of temporal layers: "
2108 << static_cast<int>(
2109 vie_codec.codecSpecific.VP8.numberOfTemporalLayers);
2110 }
2111
2112 }
2113
SetReceiveCodecs(int channel_id)2114 bool WebRtcVideoMediaChannel::SetReceiveCodecs(int channel_id) {
2115 int red_type = -1;
2116 int fec_type = -1;
2117 for (std::vector<webrtc::VideoCodec>::iterator it = receive_codecs_.begin();
2118 it != receive_codecs_.end(); ++it) {
2119 if (it->codecType == webrtc::kVideoCodecRED) {
2120 red_type = it->plType;
2121 } else if (it->codecType == webrtc::kVideoCodecULPFEC) {
2122 fec_type = it->plType;
2123 }
2124 if (engine()->vie()->codec()->SetReceiveCodec(channel_id, *it) != 0) {
2125 LOG_RTCERR2(SetReceiveCodec, channel_id, it->plName);
2126 return false;
2127 }
2128 }
2129
2130 // Enable video protection. For a sending channel, this will be taken care of
2131 // in SetSendCodecs.
2132 if (channel_id != vie_channel_) {
2133 if (!SetNackFec(channel_id, red_type, fec_type)) {
2134 return false;
2135 }
2136 }
2137
2138 // Start receiving packets if at least one receive codec has been set.
2139 if (!receive_codecs_.empty()) {
2140 if (engine()->vie()->base()->StartReceive(channel_id) != 0) {
2141 LOG_RTCERR1(StartReceive, channel_id);
2142 return false;
2143 }
2144 }
2145 return true;
2146 }
2147
2148 // If the new frame size is different from the send codec size we set on vie,
2149 // we need to reset the send codec on vie.
2150 // The new send codec size should not exceed send_codec_ which is controlled
2151 // only by the 'jec' logic.
MaybeResetVieSendCodec(int new_width,int new_height,bool * reset)2152 bool WebRtcVideoMediaChannel::MaybeResetVieSendCodec(int new_width,
2153 int new_height,
2154 bool* reset) {
2155 if (reset) {
2156 *reset = false;
2157 }
2158
2159 if (NULL == send_codec_.get()) {
2160 return false;
2161 }
2162
2163 // Vie send codec size should not exceed send_codec_.
2164 int target_width = new_width;
2165 int target_height = new_height;
2166 if (new_width > send_codec_->width || new_height > send_codec_->height) {
2167 target_width = send_codec_->width;
2168 target_height = send_codec_->height;
2169 }
2170
2171 // Get current vie codec.
2172 webrtc::VideoCodec vie_codec;
2173 if (engine()->vie()->codec()->GetSendCodec(vie_channel_, vie_codec) != 0) {
2174 LOG_RTCERR1(GetSendCodec, vie_channel_);
2175 return false;
2176 }
2177
2178 // Only reset send codec when there is a size change.
2179 if (target_width != vie_codec.width || target_height != vie_codec.height) {
2180 // Set the new codec on vie.
2181 vie_codec.width = target_width;
2182 vie_codec.height = target_height;
2183
2184 if (engine()->vie()->codec()->SetSendCodec(vie_channel_, vie_codec) != 0) {
2185 LOG_RTCERR1(SetSendCodec, vie_channel_);
2186 return false;
2187 }
2188 if (reset) {
2189 *reset = true;
2190 }
2191 LogSendCodecChange("Capture size changed");
2192 }
2193
2194 return true;
2195 }
2196
SendPacket(int channel,const void * data,int len)2197 int WebRtcVideoMediaChannel::SendPacket(int channel, const void* data,
2198 int len) {
2199 if (!network_interface_) {
2200 return -1;
2201 }
2202 talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
2203 return network_interface_->SendPacket(&packet) ? len : -1;
2204 }
2205
SendRTCPPacket(int channel,const void * data,int len)2206 int WebRtcVideoMediaChannel::SendRTCPPacket(int channel,
2207 const void* data,
2208 int len) {
2209 if (!network_interface_) {
2210 return -1;
2211 }
2212 talk_base::Buffer packet(data, len, kMaxRtpPacketLen);
2213 return network_interface_->SendRtcp(&packet) ? len : -1;
2214 }
2215
2216 } // namespace cricket
2217
2218 #endif // HAVE_WEBRTC_VIDEO
2219