1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef CALL_RTP_VIDEO_SENDER_H_ 12 #define CALL_RTP_VIDEO_SENDER_H_ 13 14 #include <map> 15 #include <memory> 16 #include <unordered_set> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/call/transport.h" 22 #include "api/fec_controller.h" 23 #include "api/fec_controller_override.h" 24 #include "api/rtc_event_log/rtc_event_log.h" 25 #include "api/transport/field_trial_based_config.h" 26 #include "api/video_codecs/video_encoder.h" 27 #include "call/rtp_config.h" 28 #include "call/rtp_payload_params.h" 29 #include "call/rtp_transport_controller_send_interface.h" 30 #include "call/rtp_video_sender_interface.h" 31 #include "modules/rtp_rtcp/include/flexfec_sender.h" 32 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 33 #include "modules/rtp_rtcp/source/rtp_sender.h" 34 #include "modules/rtp_rtcp/source/rtp_sender_video.h" 35 #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" 36 #include "modules/rtp_rtcp/source/rtp_video_header.h" 37 #include "modules/utility/include/process_thread.h" 38 #include "rtc_base/constructor_magic.h" 39 #include "rtc_base/rate_limiter.h" 40 #include "rtc_base/synchronization/mutex.h" 41 #include "rtc_base/thread_annotations.h" 42 #include "rtc_base/thread_checker.h" 43 44 namespace webrtc { 45 46 class FrameEncryptorInterface; 47 class RtpTransportControllerSendInterface; 48 49 namespace webrtc_internal_rtp_video_sender { 50 // RTP state for a single simulcast stream. Internal to the implementation of 51 // RtpVideoSender. 52 struct RtpStreamSender { 53 RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, 54 std::unique_ptr<RTPSenderVideo> sender_video, 55 std::unique_ptr<VideoFecGenerator> fec_generator); 56 ~RtpStreamSender(); 57 58 RtpStreamSender(RtpStreamSender&&) = default; 59 RtpStreamSender& operator=(RtpStreamSender&&) = default; 60 61 // Note: Needs pointer stability. 62 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp; 63 std::unique_ptr<RTPSenderVideo> sender_video; 64 std::unique_ptr<VideoFecGenerator> fec_generator; 65 }; 66 67 } // namespace webrtc_internal_rtp_video_sender 68 69 // RtpVideoSender routes outgoing data to the correct sending RTP module, based 70 // on the simulcast layer in RTPVideoHeader. 71 class RtpVideoSender : public RtpVideoSenderInterface, 72 public VCMProtectionCallback, 73 public StreamFeedbackObserver { 74 public: 75 // Rtp modules are assumed to be sorted in simulcast index order. 76 RtpVideoSender( 77 Clock* clock, 78 std::map<uint32_t, RtpState> suspended_ssrcs, 79 const std::map<uint32_t, RtpPayloadState>& states, 80 const RtpConfig& rtp_config, 81 int rtcp_report_interval_ms, 82 Transport* send_transport, 83 const RtpSenderObservers& observers, 84 RtpTransportControllerSendInterface* transport, 85 RtcEventLog* event_log, 86 RateLimiter* retransmission_limiter, // move inside RtpTransport 87 std::unique_ptr<FecController> fec_controller, 88 FrameEncryptorInterface* frame_encryptor, 89 const CryptoOptions& crypto_options, // move inside RtpTransport 90 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 91 ~RtpVideoSender() override; 92 93 // RegisterProcessThread register |module_process_thread| with those objects 94 // that use it. Registration has to happen on the thread were 95 // |module_process_thread| was created (libjingle's worker thread). 96 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, 97 // maybe |worker_queue|. 98 void RegisterProcessThread(ProcessThread* module_process_thread) 99 RTC_LOCKS_EXCLUDED(mutex_) override; 100 void DeRegisterProcessThread() RTC_LOCKS_EXCLUDED(mutex_) override; 101 102 // RtpVideoSender will only route packets if being active, all packets will be 103 // dropped otherwise. 104 void SetActive(bool active) RTC_LOCKS_EXCLUDED(mutex_) override; 105 // Sets the sending status of the rtp modules and appropriately sets the 106 // payload router to active if any rtp modules are active. 107 void SetActiveModules(const std::vector<bool> active_modules) 108 RTC_LOCKS_EXCLUDED(mutex_) override; 109 bool IsActive() RTC_LOCKS_EXCLUDED(mutex_) override; 110 111 void OnNetworkAvailability(bool network_available) 112 RTC_LOCKS_EXCLUDED(mutex_) override; 113 std::map<uint32_t, RtpState> GetRtpStates() const 114 RTC_LOCKS_EXCLUDED(mutex_) override; 115 std::map<uint32_t, RtpPayloadState> GetRtpPayloadStates() const 116 RTC_LOCKS_EXCLUDED(mutex_) override; 117 118 void DeliverRtcp(const uint8_t* packet, size_t length) 119 RTC_LOCKS_EXCLUDED(mutex_) override; 120 121 // Implements webrtc::VCMProtectionCallback. 122 int ProtectionRequest(const FecProtectionParams* delta_params, 123 const FecProtectionParams* key_params, 124 uint32_t* sent_video_rate_bps, 125 uint32_t* sent_nack_rate_bps, 126 uint32_t* sent_fec_rate_bps) 127 RTC_LOCKS_EXCLUDED(mutex_) override; 128 129 // Implements FecControllerOverride. 130 void SetFecAllowed(bool fec_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; 131 132 // Implements EncodedImageCallback. 133 // Returns 0 if the packet was routed / sent, -1 otherwise. 134 EncodedImageCallback::Result OnEncodedImage( 135 const EncodedImage& encoded_image, 136 const CodecSpecificInfo* codec_specific_info) 137 RTC_LOCKS_EXCLUDED(mutex_) override; 138 139 void OnBitrateAllocationUpdated(const VideoBitrateAllocation& bitrate) 140 RTC_LOCKS_EXCLUDED(mutex_) override; 141 void OnVideoLayersAllocationUpdated( 142 const VideoLayersAllocation& layers) override; 143 void OnTransportOverheadChanged(size_t transport_overhead_bytes_per_packet) 144 RTC_LOCKS_EXCLUDED(mutex_) override; 145 void OnBitrateUpdated(BitrateAllocationUpdate update, int framerate) 146 RTC_LOCKS_EXCLUDED(mutex_) override; 147 uint32_t GetPayloadBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 148 uint32_t GetProtectionBitrateBps() const RTC_LOCKS_EXCLUDED(mutex_) override; 149 void SetEncodingData(size_t width, size_t height, size_t num_temporal_layers) 150 RTC_LOCKS_EXCLUDED(mutex_) override; 151 152 std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( 153 uint32_t ssrc, 154 rtc::ArrayView<const uint16_t> sequence_numbers) const 155 RTC_LOCKS_EXCLUDED(mutex_) override; 156 157 // From StreamFeedbackObserver. 158 void OnPacketFeedbackVector( 159 std::vector<StreamPacketInfo> packet_feedback_vector) 160 RTC_LOCKS_EXCLUDED(mutex_) override; 161 162 private: 163 bool IsActiveLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 164 void SetActiveModulesLocked(const std::vector<bool> active_modules) 165 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 166 void UpdateModuleSendingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 167 void ConfigureProtection(); 168 void ConfigureSsrcs(); 169 void ConfigureRids(); 170 bool NackEnabled() const; 171 uint32_t GetPacketizationOverheadRate() const; 172 173 const FieldTrialBasedConfig field_trials_; 174 const bool send_side_bwe_with_overhead_; 175 const bool has_packet_feedback_; 176 177 // TODO(holmer): Remove mutex_ once RtpVideoSender runs on the 178 // transport task queue. 179 mutable Mutex mutex_; 180 bool active_ RTC_GUARDED_BY(mutex_); 181 182 ProcessThread* module_process_thread_; 183 rtc::ThreadChecker module_process_thread_checker_; 184 std::map<uint32_t, RtpState> suspended_ssrcs_; 185 186 const std::unique_ptr<FecController> fec_controller_; 187 bool fec_allowed_ RTC_GUARDED_BY(mutex_); 188 189 // Rtp modules are assumed to be sorted in simulcast index order. 190 const std::vector<webrtc_internal_rtp_video_sender::RtpStreamSender> 191 rtp_streams_; 192 const RtpConfig rtp_config_; 193 const absl::optional<VideoCodecType> codec_type_; 194 RtpTransportControllerSendInterface* const transport_; 195 196 // When using the generic descriptor we want all simulcast streams to share 197 // one frame id space (so that the SFU can switch stream without having to 198 // rewrite the frame id), therefore |shared_frame_id| has to live in a place 199 // where we are aware of all the different streams. 200 int64_t shared_frame_id_ = 0; 201 std::vector<RtpPayloadParams> params_ RTC_GUARDED_BY(mutex_); 202 203 size_t transport_overhead_bytes_per_packet_ RTC_GUARDED_BY(mutex_); 204 uint32_t protection_bitrate_bps_; 205 uint32_t encoder_target_rate_bps_; 206 207 std::vector<bool> loss_mask_vector_ RTC_GUARDED_BY(mutex_); 208 209 std::vector<FrameCounts> frame_counts_ RTC_GUARDED_BY(mutex_); 210 FrameCountObserver* const frame_count_observer_; 211 212 // Effectively const map from SSRC to RtpRtcp, for all media SSRCs. 213 // This map is set at construction time and never changed, but it's 214 // non-trivial to make it properly const. 215 std::map<uint32_t, RtpRtcpInterface*> ssrc_to_rtp_module_; 216 217 RTC_DISALLOW_COPY_AND_ASSIGN(RtpVideoSender); 218 }; 219 220 } // namespace webrtc 221 222 #endif // CALL_RTP_VIDEO_SENDER_H_ 223