1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_ 12 #define MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_ 13 14 #include <map> 15 #include <memory> 16 #include <set> 17 #include <string> 18 #include <vector> 19 20 #include "api/call/transport.h" 21 #include "api/optional.h" 22 #include "api/video/video_frame.h" 23 #include "api/video_codecs/sdp_video_format.h" 24 #include "call/call.h" 25 #include "call/flexfec_receive_stream.h" 26 #include "call/video_receive_stream.h" 27 #include "call/video_send_stream.h" 28 #include "media/base/mediaengine.h" 29 #include "media/base/videosinkinterface.h" 30 #include "media/base/videosourceinterface.h" 31 #include "media/engine/webrtcvideodecoderfactory.h" 32 #include "media/engine/webrtcvideoencoderfactory.h" 33 #include "rtc_base/asyncinvoker.h" 34 #include "rtc_base/criticalsection.h" 35 #include "rtc_base/networkroute.h" 36 #include "rtc_base/thread_annotations.h" 37 #include "rtc_base/thread_checker.h" 38 39 namespace webrtc { 40 class VideoDecoder; 41 class VideoDecoderFactory; 42 class VideoEncoder; 43 class VideoEncoderFactory; 44 struct MediaConfig; 45 } 46 47 namespace rtc { 48 class Thread; 49 } // namespace rtc 50 51 namespace cricket { 52 53 class DecoderFactoryAdapter; 54 class VideoCapturer; 55 class VideoProcessor; 56 class VideoRenderer; 57 class VoiceMediaChannel; 58 class WebRtcDecoderObserver; 59 class WebRtcEncoderObserver; 60 class WebRtcLocalStreamInfo; 61 class WebRtcRenderAdapter; 62 class WebRtcVideoChannel; 63 class WebRtcVideoChannelRecvInfo; 64 class WebRtcVideoChannelSendInfo; 65 class WebRtcVoiceEngine; 66 class WebRtcVoiceMediaChannel; 67 68 class UnsignalledSsrcHandler { 69 public: 70 enum Action { 71 kDropPacket, 72 kDeliverPacket, 73 }; 74 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, 75 uint32_t ssrc) = 0; 76 virtual ~UnsignalledSsrcHandler() = default; 77 }; 78 79 // TODO(pbos): Remove, use external handlers only. 80 class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler { 81 public: 82 DefaultUnsignalledSsrcHandler(); 83 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, 84 uint32_t ssrc) override; 85 86 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const; 87 void SetDefaultSink(WebRtcVideoChannel* channel, 88 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 89 90 virtual ~DefaultUnsignalledSsrcHandler() = default; 91 92 private: 93 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_; 94 }; 95 96 // WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). 97 class WebRtcVideoEngine { 98 public: 99 // Internal SW video codecs will be added on top of the external codecs. 100 WebRtcVideoEngine( 101 std::unique_ptr<WebRtcVideoEncoderFactory> external_video_encoder_factory, 102 std::unique_ptr<WebRtcVideoDecoderFactory> 103 external_video_decoder_factory); 104 105 // These video codec factories represents all video codecs, i.e. both software 106 // and external hardware codecs. 107 WebRtcVideoEngine( 108 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, 109 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory); 110 111 virtual ~WebRtcVideoEngine(); 112 113 WebRtcVideoChannel* CreateChannel(webrtc::Call* call, 114 const MediaConfig& config, 115 const VideoOptions& options); 116 117 std::vector<VideoCodec> codecs() const; 118 RtpCapabilities GetCapabilities() const; 119 120 private: 121 const std::unique_ptr<DecoderFactoryAdapter> decoder_factory_; 122 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_; 123 }; 124 125 class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { 126 public: 127 WebRtcVideoChannel(webrtc::Call* call, 128 const MediaConfig& config, 129 const VideoOptions& options, 130 webrtc::VideoEncoderFactory* encoder_factory, 131 DecoderFactoryAdapter* decoder_factory); 132 ~WebRtcVideoChannel() override; 133 134 // VideoMediaChannel implementation 135 rtc::DiffServCodePoint PreferredDscp() const override; 136 137 bool SetSendParameters(const VideoSendParameters& params) override; 138 bool SetRecvParameters(const VideoRecvParameters& params) override; 139 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; 140 bool SetRtpSendParameters(uint32_t ssrc, 141 const webrtc::RtpParameters& parameters) override; 142 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override; 143 bool SetRtpReceiveParameters( 144 uint32_t ssrc, 145 const webrtc::RtpParameters& parameters) override; 146 bool GetSendCodec(VideoCodec* send_codec) override; 147 bool SetSend(bool send) override; 148 bool SetVideoSend( 149 uint32_t ssrc, 150 bool enable, 151 const VideoOptions* options, 152 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; 153 bool AddSendStream(const StreamParams& sp) override; 154 bool RemoveSendStream(uint32_t ssrc) override; 155 bool AddRecvStream(const StreamParams& sp) override; 156 bool AddRecvStream(const StreamParams& sp, bool default_stream); 157 bool RemoveRecvStream(uint32_t ssrc) override; 158 bool SetSink(uint32_t ssrc, 159 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 160 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; 161 bool GetStats(VideoMediaInfo* info) override; 162 163 void OnPacketReceived(rtc::CopyOnWriteBuffer* packet, 164 const rtc::PacketTime& packet_time) override; 165 void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet, 166 const rtc::PacketTime& packet_time) override; 167 void OnReadyToSend(bool ready) override; 168 void OnNetworkRouteChanged(const std::string& transport_name, 169 const rtc::NetworkRoute& network_route) override; 170 void SetInterface(NetworkInterface* iface) override; 171 172 // Implemented for VideoMediaChannelTest. sending()173 bool sending() const { return sending_; } 174 175 rtc::Optional<uint32_t> GetDefaultReceiveStreamSsrc(); 176 177 // AdaptReason is used for expressing why a WebRtcVideoSendStream request 178 // a lower input frame size than the currently configured camera input frame 179 // size. There can be more than one reason OR:ed together. 180 enum AdaptReason { 181 ADAPTREASON_NONE = 0, 182 ADAPTREASON_CPU = 1, 183 ADAPTREASON_BANDWIDTH = 2, 184 }; 185 186 static constexpr int kDefaultQpMax = 56; 187 188 private: 189 class WebRtcVideoReceiveStream; 190 struct VideoCodecSettings { 191 VideoCodecSettings(); 192 193 // Checks if all members of |*this| are equal to the corresponding members 194 // of |other|. 195 bool operator==(const VideoCodecSettings& other) const; 196 bool operator!=(const VideoCodecSettings& other) const; 197 198 // Checks if all members of |a|, except |flexfec_payload_type|, are equal 199 // to the corresponding members of |b|. 200 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, 201 const VideoCodecSettings& b); 202 203 VideoCodec codec; 204 webrtc::UlpfecConfig ulpfec; 205 int flexfec_payload_type; 206 int rtx_payload_type; 207 }; 208 209 struct ChangedSendParameters { 210 // These optionals are unset if not changed. 211 rtc::Optional<VideoCodecSettings> codec; 212 rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 213 rtc::Optional<int> max_bandwidth_bps; 214 rtc::Optional<bool> conference_mode; 215 rtc::Optional<webrtc::RtcpMode> rtcp_mode; 216 }; 217 218 struct ChangedRecvParameters { 219 // These optionals are unset if not changed. 220 rtc::Optional<std::vector<VideoCodecSettings>> codec_settings; 221 rtc::Optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; 222 // Keep track of the FlexFEC payload type separately from |codec_settings|. 223 // This allows us to recreate the FlexfecReceiveStream separately from the 224 // VideoReceiveStream when the FlexFEC payload type is changed. 225 rtc::Optional<int> flexfec_payload_type; 226 }; 227 228 bool GetChangedSendParameters(const VideoSendParameters& params, 229 ChangedSendParameters* changed_params) const; 230 bool GetChangedRecvParameters(const VideoRecvParameters& params, 231 ChangedRecvParameters* changed_params) const; 232 233 void SetMaxSendBandwidth(int bps); 234 235 void ConfigureReceiverRtp( 236 webrtc::VideoReceiveStream::Config* config, 237 webrtc::FlexfecReceiveStream::Config* flexfec_config, 238 const StreamParams& sp) const; 239 bool ValidateSendSsrcAvailability(const StreamParams& sp) const 240 RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 241 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const 242 RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 243 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) 244 RTC_EXCLUSIVE_LOCKS_REQUIRED(stream_crit_); 245 246 static std::string CodecSettingsVectorToString( 247 const std::vector<VideoCodecSettings>& codecs); 248 249 // Wrapper for the sender part. 250 class WebRtcVideoSendStream 251 : public rtc::VideoSourceInterface<webrtc::VideoFrame> { 252 public: 253 WebRtcVideoSendStream( 254 webrtc::Call* call, 255 const StreamParams& sp, 256 webrtc::VideoSendStream::Config config, 257 const VideoOptions& options, 258 webrtc::VideoEncoderFactory* encoder_factory, 259 bool enable_cpu_overuse_detection, 260 int max_bitrate_bps, 261 const rtc::Optional<VideoCodecSettings>& codec_settings, 262 const rtc::Optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, 263 const VideoSendParameters& send_params); 264 virtual ~WebRtcVideoSendStream(); 265 266 void SetSendParameters(const ChangedSendParameters& send_params); 267 bool SetRtpParameters(const webrtc::RtpParameters& parameters); 268 webrtc::RtpParameters GetRtpParameters() const; 269 270 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>. 271 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream 272 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to 273 // the worker thread. 274 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, 275 const rtc::VideoSinkWants& wants) override; 276 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; 277 278 bool SetVideoSend(bool mute, 279 const VideoOptions* options, 280 rtc::VideoSourceInterface<webrtc::VideoFrame>* source); 281 282 void SetSend(bool send); 283 284 const std::vector<uint32_t>& GetSsrcs() const; 285 VideoSenderInfo GetVideoSenderInfo(bool log_stats); 286 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); 287 288 private: 289 // Parameters needed to reconstruct the underlying stream. 290 // webrtc::VideoSendStream doesn't support setting a lot of options on the 291 // fly, so when those need to be changed we tear down and reconstruct with 292 // similar parameters depending on which options changed etc. 293 struct VideoSendStreamParameters { 294 VideoSendStreamParameters( 295 webrtc::VideoSendStream::Config config, 296 const VideoOptions& options, 297 int max_bitrate_bps, 298 const rtc::Optional<VideoCodecSettings>& codec_settings); 299 webrtc::VideoSendStream::Config config; 300 VideoOptions options; 301 int max_bitrate_bps; 302 bool conference_mode; 303 rtc::Optional<VideoCodecSettings> codec_settings; 304 // Sent resolutions + bitrates etc. by the underlying VideoSendStream, 305 // typically changes when setting a new resolution or reconfiguring 306 // bitrates. 307 webrtc::VideoEncoderConfig encoder_config; 308 }; 309 310 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> 311 ConfigureVideoEncoderSettings(const VideoCodec& codec); 312 void SetCodec(const VideoCodecSettings& codec, 313 bool force_encoder_allocation); 314 void RecreateWebRtcStream(); 315 webrtc::VideoEncoderConfig CreateVideoEncoderConfig( 316 const VideoCodec& codec) const; 317 void ReconfigureEncoder(); 318 bool ValidateRtpParameters(const webrtc::RtpParameters& parameters); 319 320 // Calls Start or Stop according to whether or not |sending_| is true, 321 // and whether or not the encoding in |rtp_parameters_| is active. 322 void UpdateSendState(); 323 324 webrtc::VideoSendStream::DegradationPreference GetDegradationPreference() 325 const RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); 326 327 rtc::ThreadChecker thread_checker_; 328 rtc::AsyncInvoker invoker_; 329 rtc::Thread* worker_thread_; 330 const std::vector<uint32_t> ssrcs_ RTC_ACCESS_ON(&thread_checker_); 331 const std::vector<SsrcGroup> ssrc_groups_ RTC_ACCESS_ON(&thread_checker_); 332 webrtc::Call* const call_; 333 const bool enable_cpu_overuse_detection_; 334 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_ 335 RTC_ACCESS_ON(&thread_checker_); 336 webrtc::VideoEncoderFactory* const encoder_factory_ 337 RTC_ACCESS_ON(&thread_checker_); 338 339 webrtc::VideoSendStream* stream_ RTC_ACCESS_ON(&thread_checker_); 340 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_ 341 RTC_ACCESS_ON(&thread_checker_); 342 // Contains settings that are the same for all streams in the MediaChannel, 343 // such as codecs, header extensions, and the global bitrate limit for the 344 // entire channel. 345 VideoSendStreamParameters parameters_ RTC_ACCESS_ON(&thread_checker_); 346 // Contains settings that are unique for each stream, such as max_bitrate. 347 // Does *not* contain codecs, however. 348 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. 349 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only 350 // one stream per MediaChannel. 351 webrtc::RtpParameters rtp_parameters_ RTC_ACCESS_ON(&thread_checker_); 352 std::unique_ptr<webrtc::VideoEncoder> allocated_encoder_ 353 RTC_ACCESS_ON(&thread_checker_); 354 VideoCodec allocated_codec_ RTC_ACCESS_ON(&thread_checker_); 355 356 bool sending_ RTC_ACCESS_ON(&thread_checker_); 357 }; 358 359 // Wrapper for the receiver part, contains configs etc. that are needed to 360 // reconstruct the underlying VideoReceiveStream. 361 class WebRtcVideoReceiveStream 362 : public rtc::VideoSinkInterface<webrtc::VideoFrame> { 363 public: 364 WebRtcVideoReceiveStream( 365 webrtc::Call* call, 366 const StreamParams& sp, 367 webrtc::VideoReceiveStream::Config config, 368 DecoderFactoryAdapter* decoder_factory, 369 bool default_stream, 370 const std::vector<VideoCodecSettings>& recv_codecs, 371 const webrtc::FlexfecReceiveStream::Config& flexfec_config); 372 ~WebRtcVideoReceiveStream(); 373 374 const std::vector<uint32_t>& GetSsrcs() const; 375 rtc::Optional<uint32_t> GetFirstPrimarySsrc() const; 376 377 void SetLocalSsrc(uint32_t local_ssrc); 378 // TODO(deadbeef): Move these feedback parameters into the recv parameters. 379 void SetFeedbackParameters(bool nack_enabled, 380 bool remb_enabled, 381 bool transport_cc_enabled, 382 webrtc::RtcpMode rtcp_mode); 383 void SetRecvParameters(const ChangedRecvParameters& recv_params); 384 385 void OnFrame(const webrtc::VideoFrame& frame) override; 386 bool IsDefaultStream() const; 387 388 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); 389 390 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); 391 392 private: 393 struct SdpVideoFormatCompare { operatorSdpVideoFormatCompare394 bool operator()(const webrtc::SdpVideoFormat& lhs, 395 const webrtc::SdpVideoFormat& rhs) const { 396 return std::tie(lhs.name, lhs.parameters) < 397 std::tie(rhs.name, rhs.parameters); 398 } 399 }; 400 typedef std::map<webrtc::SdpVideoFormat, 401 std::unique_ptr<webrtc::VideoDecoder>, 402 SdpVideoFormatCompare> 403 DecoderMap; 404 405 void RecreateWebRtcVideoStream(); 406 void MaybeRecreateWebRtcFlexfecStream(); 407 408 void MaybeAssociateFlexfecWithVideo(); 409 void MaybeDissociateFlexfecFromVideo(); 410 411 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs, 412 DecoderMap* old_codecs); 413 void ConfigureFlexfecCodec(int flexfec_payload_type); 414 415 std::string GetCodecNameFromPayloadType(int payload_type); 416 417 webrtc::Call* const call_; 418 StreamParams stream_params_; 419 420 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are 421 // destroyed by calling call_->DestroyVideoReceiveStream and 422 // call_->DestroyFlexfecReceiveStream, respectively. 423 webrtc::VideoReceiveStream* stream_; 424 const bool default_stream_; 425 webrtc::VideoReceiveStream::Config config_; 426 webrtc::FlexfecReceiveStream::Config flexfec_config_; 427 webrtc::FlexfecReceiveStream* flexfec_stream_; 428 429 DecoderFactoryAdapter* decoder_factory_; 430 DecoderMap allocated_decoders_; 431 432 rtc::CriticalSection sink_lock_; 433 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ 434 RTC_GUARDED_BY(sink_lock_); 435 // Expands remote RTP timestamps to int64_t to be able to estimate how long 436 // the stream has been running. 437 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_ 438 RTC_GUARDED_BY(sink_lock_); 439 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); 440 // Start NTP time is estimated as current remote NTP time (estimated from 441 // RTCP) minus the elapsed time, as soon as remote NTP time is available. 442 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); 443 }; 444 445 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine); 446 447 bool SendRtp(const uint8_t* data, 448 size_t len, 449 const webrtc::PacketOptions& options) override; 450 bool SendRtcp(const uint8_t* data, size_t len) override; 451 452 static std::vector<VideoCodecSettings> MapCodecs( 453 const std::vector<VideoCodec>& codecs); 454 // Select what video codec will be used for sending, i.e. what codec is used 455 // for local encoding, based on supported remote codecs. The first remote 456 // codec that is supported locally will be selected. 457 rtc::Optional<VideoCodecSettings> SelectSendVideoCodec( 458 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const; 459 460 static bool NonFlexfecReceiveCodecsHaveChanged( 461 std::vector<VideoCodecSettings> before, 462 std::vector<VideoCodecSettings> after); 463 464 void FillSenderStats(VideoMediaInfo* info, bool log_stats); 465 void FillReceiverStats(VideoMediaInfo* info, bool log_stats); 466 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, 467 VideoMediaInfo* info); 468 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info); 469 470 rtc::ThreadChecker thread_checker_; 471 472 uint32_t rtcp_receiver_report_ssrc_; 473 bool sending_; 474 webrtc::Call* const call_; 475 476 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_; 477 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_; 478 479 const MediaConfig::Video video_config_; 480 481 rtc::CriticalSection stream_crit_; 482 // Using primary-ssrc (first ssrc) as key. 483 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ 484 RTC_GUARDED_BY(stream_crit_); 485 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ 486 RTC_GUARDED_BY(stream_crit_); 487 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(stream_crit_); 488 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(stream_crit_); 489 490 rtc::Optional<VideoCodecSettings> send_codec_; 491 rtc::Optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_; 492 493 webrtc::VideoEncoderFactory* const encoder_factory_; 494 DecoderFactoryAdapter* const decoder_factory_; 495 std::vector<VideoCodecSettings> recv_codecs_; 496 std::vector<webrtc::RtpExtension> recv_rtp_extensions_; 497 // See reason for keeping track of the FlexFEC payload type separately in 498 // comment in WebRtcVideoChannel::ChangedRecvParameters. 499 int recv_flexfec_payload_type_; 500 webrtc::Call::Config::BitrateConfig bitrate_config_; 501 // TODO(deadbeef): Don't duplicate information between 502 // send_params/recv_params, rtp_extensions, options, etc. 503 VideoSendParameters send_params_; 504 VideoOptions default_send_options_; 505 VideoRecvParameters recv_params_; 506 int64_t last_stats_log_ms_; 507 }; 508 509 class EncoderStreamFactory 510 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface { 511 public: 512 EncoderStreamFactory(std::string codec_name, 513 int max_qp, 514 int max_framerate, 515 bool is_screencast, 516 bool conference_mode); 517 518 private: 519 std::vector<webrtc::VideoStream> CreateEncoderStreams( 520 int width, 521 int height, 522 const webrtc::VideoEncoderConfig& encoder_config) override; 523 524 const std::string codec_name_; 525 const int max_qp_; 526 const int max_framerate_; 527 const bool is_screencast_; 528 const bool conference_mode_; 529 }; 530 531 } // namespace cricket 532 533 #endif // MEDIA_ENGINE_WEBRTCVIDEOENGINE_H_ 534