1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 12 #define MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 13 14 #include <map> 15 #include <memory> 16 #include <vector> 17 18 #include "absl/strings/string_view.h" 19 #include "absl/types/optional.h" 20 #include "api/array_view.h" 21 #include "api/frame_transformer_interface.h" 22 #include "api/scoped_refptr.h" 23 #include "api/task_queue/task_queue_base.h" 24 #include "api/transport/rtp/dependency_descriptor.h" 25 #include "api/video/video_codec_type.h" 26 #include "api/video/video_frame_type.h" 27 #include "modules/include/module_common_types.h" 28 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 29 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h" 30 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" 31 #include "modules/rtp_rtcp/source/rtp_sender.h" 32 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" 33 #include "modules/rtp_rtcp/source/rtp_video_header.h" 34 #include "modules/rtp_rtcp/source/video_fec_generator.h" 35 #include "rtc_base/critical_section.h" 36 #include "rtc_base/one_time_event.h" 37 #include "rtc_base/race_checker.h" 38 #include "rtc_base/rate_statistics.h" 39 #include "rtc_base/synchronization/sequence_checker.h" 40 #include "rtc_base/thread_annotations.h" 41 42 namespace webrtc { 43 44 class FrameEncryptorInterface; 45 class RtpPacketizer; 46 class RtpPacketToSend; 47 48 // kConditionallyRetransmitHigherLayers allows retransmission of video frames 49 // in higher layers if either the last frame in that layer was too far back in 50 // time, or if we estimate that a new frame will be available in a lower layer 51 // in a shorter time than it would take to request and receive a retransmission. 52 enum RetransmissionMode : uint8_t { 53 kRetransmitOff = 0x0, 54 kRetransmitBaseLayer = 0x2, 55 kRetransmitHigherLayers = 0x4, 56 kRetransmitAllLayers = 0x6, 57 kConditionallyRetransmitHigherLayers = 0x8 58 }; 59 60 class RTPSenderVideo { 61 public: 62 static constexpr int64_t kTLRateWindowSizeMs = 2500; 63 64 struct Config { 65 Config() = default; 66 Config(const Config&) = delete; 67 Config(Config&&) = default; 68 69 // All members of this struct, with the exception of |field_trials|, are 70 // expected to outlive the RTPSenderVideo object they are passed to. 71 Clock* clock = nullptr; 72 RTPSender* rtp_sender = nullptr; 73 FlexfecSender* flexfec_sender = nullptr; 74 VideoFecGenerator* fec_generator = nullptr; 75 FrameEncryptorInterface* frame_encryptor = nullptr; 76 bool require_frame_encryption = false; 77 bool enable_retransmit_all_layers = false; 78 absl::optional<int> red_payload_type; 79 const WebRtcKeyValueConfig* field_trials = nullptr; 80 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; 81 TaskQueueBase* worker_queue = nullptr; 82 }; 83 84 explicit RTPSenderVideo(const Config& config); 85 86 virtual ~RTPSenderVideo(); 87 88 // expected_retransmission_time_ms.has_value() -> retransmission allowed. 89 // Calls to this method is assumed to be externally serialized. 90 bool SendVideo(int payload_type, 91 absl::optional<VideoCodecType> codec_type, 92 uint32_t rtp_timestamp, 93 int64_t capture_time_ms, 94 rtc::ArrayView<const uint8_t> payload, 95 const RTPFragmentationHeader* fragmentation, 96 RTPVideoHeader video_header, 97 absl::optional<int64_t> expected_retransmission_time_ms); 98 99 bool SendEncodedImage( 100 int payload_type, 101 absl::optional<VideoCodecType> codec_type, 102 uint32_t rtp_timestamp, 103 const EncodedImage& encoded_image, 104 const RTPFragmentationHeader* fragmentation, 105 RTPVideoHeader video_header, 106 absl::optional<int64_t> expected_retransmission_time_ms); 107 108 // Configures video structures produced by encoder to send using the 109 // dependency descriptor rtp header extension. Next call to SendVideo should 110 // have video_header.frame_type == kVideoFrameKey. 111 // All calls to SendVideo after this call must use video_header compatible 112 // with the video_structure. 113 void SetVideoStructure(const FrameDependencyStructure* video_structure); 114 void SetVideoStructureUnderLock( 115 const FrameDependencyStructure* video_structure); 116 117 // FlexFEC/ULPFEC. 118 // Set FEC rates, max frames before FEC is sent, and type of FEC masks. 119 void SetFecParameters(const FecProtectionParams& delta_params, 120 const FecProtectionParams& key_params); 121 122 uint32_t VideoBitrateSent() const; 123 uint32_t FecOverheadRate() const; 124 125 // Returns the current packetization overhead rate, in bps. Note that this is 126 // the payload overhead, eg the VP8 payload headers, not the RTP headers 127 // or extension/ 128 uint32_t PacketizationOverheadBps() const; 129 130 protected: 131 static uint8_t GetTemporalId(const RTPVideoHeader& header); 132 bool AllowRetransmission(uint8_t temporal_id, 133 int32_t retransmission_settings, 134 int64_t expected_retransmission_time_ms); 135 136 private: 137 struct TemporalLayerStats { TemporalLayerStatsTemporalLayerStats138 TemporalLayerStats() 139 : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000), 140 last_frame_time_ms(0) {} 141 // Frame rate, in frames per 1000 seconds. This essentially turns the fps 142 // value into a fixed point value with three decimals. Improves precision at 143 // low frame rates. 144 RateStatistics frame_rate_fp1000s; 145 int64_t last_frame_time_ms; 146 }; 147 148 void AddRtpHeaderExtensions( 149 const RTPVideoHeader& video_header, 150 const absl::optional<AbsoluteCaptureTime>& absolute_capture_time, 151 bool first_packet, 152 bool last_packet, 153 RtpPacketToSend* packet) const 154 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 155 156 size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 157 158 void LogAndSendToNetwork( 159 std::vector<std::unique_ptr<RtpPacketToSend>> packets, 160 size_t unpacketized_payload_size); 161 red_enabled()162 bool red_enabled() const { return red_payload_type_.has_value(); } 163 164 bool UpdateConditionalRetransmit(uint8_t temporal_id, 165 int64_t expected_retransmission_time_ms) 166 RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_); 167 168 void MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader& header) 169 RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); 170 171 RTPSender* const rtp_sender_; 172 Clock* const clock_; 173 174 const int32_t retransmission_settings_; 175 176 // These members should only be accessed from within SendVideo() to avoid 177 // potential race conditions. 178 rtc::RaceChecker send_checker_; 179 VideoRotation last_rotation_ RTC_GUARDED_BY(send_checker_); 180 absl::optional<ColorSpace> last_color_space_ RTC_GUARDED_BY(send_checker_); 181 bool transmit_color_space_next_frame_ RTC_GUARDED_BY(send_checker_); 182 std::unique_ptr<FrameDependencyStructure> video_structure_ 183 RTC_GUARDED_BY(send_checker_); 184 185 // Current target playout delay. 186 PlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_); 187 // Flag indicating if we need to propagate |current_playout_delay_| in order 188 // to guarantee it gets delivered. 189 bool playout_delay_pending_; 190 191 // Should never be held when calling out of this class. 192 rtc::CriticalSection crit_; 193 194 const absl::optional<int> red_payload_type_; 195 VideoFecGenerator* const fec_generator_; 196 197 rtc::CriticalSection stats_crit_; 198 // Bitrate used for video payload and RTP headers. 199 RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_); 200 RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_); 201 202 std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_ 203 RTC_GUARDED_BY(stats_crit_); 204 205 OneTimeEvent first_frame_sent_; 206 207 // E2EE Custom Video Frame Encryptor (optional) 208 FrameEncryptorInterface* const frame_encryptor_ = nullptr; 209 // If set to true will require all outgoing frames to pass through an 210 // initialized frame_encryptor_ before being sent out of the network. 211 // Otherwise these payloads will be dropped. 212 const bool require_frame_encryption_; 213 // Set to true if the generic descriptor should be authenticated. 214 const bool generic_descriptor_auth_experiment_; 215 216 const bool exclude_transport_sequence_number_from_fec_experiment_; 217 218 AbsoluteCaptureTimeSender absolute_capture_time_sender_; 219 220 const rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> 221 frame_transformer_delegate_; 222 }; 223 224 } // namespace webrtc 225 226 #endif // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_ 227