1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_
12 #define MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_
13 
14 #include <map>
15 #include <memory>
16 #include <vector>
17 
18 #include "absl/strings/string_view.h"
19 #include "absl/types/optional.h"
20 #include "api/array_view.h"
21 #include "api/frame_transformer_interface.h"
22 #include "api/scoped_refptr.h"
23 #include "api/task_queue/task_queue_base.h"
24 #include "api/transport/rtp/dependency_descriptor.h"
25 #include "api/video/video_codec_type.h"
26 #include "api/video/video_frame_type.h"
27 #include "modules/include/module_common_types.h"
28 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
29 #include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
30 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
31 #include "modules/rtp_rtcp/source/rtp_sender.h"
32 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
33 #include "modules/rtp_rtcp/source/rtp_video_header.h"
34 #include "modules/rtp_rtcp/source/video_fec_generator.h"
35 #include "rtc_base/critical_section.h"
36 #include "rtc_base/one_time_event.h"
37 #include "rtc_base/race_checker.h"
38 #include "rtc_base/rate_statistics.h"
39 #include "rtc_base/synchronization/sequence_checker.h"
40 #include "rtc_base/thread_annotations.h"
41 
42 namespace webrtc {
43 
44 class FrameEncryptorInterface;
45 class RtpPacketizer;
46 class RtpPacketToSend;
47 
48 // kConditionallyRetransmitHigherLayers allows retransmission of video frames
49 // in higher layers if either the last frame in that layer was too far back in
50 // time, or if we estimate that a new frame will be available in a lower layer
51 // in a shorter time than it would take to request and receive a retransmission.
52 enum RetransmissionMode : uint8_t {
53   kRetransmitOff = 0x0,
54   kRetransmitBaseLayer = 0x2,
55   kRetransmitHigherLayers = 0x4,
56   kRetransmitAllLayers = 0x6,
57   kConditionallyRetransmitHigherLayers = 0x8
58 };
59 
60 class RTPSenderVideo {
61  public:
62   static constexpr int64_t kTLRateWindowSizeMs = 2500;
63 
64   struct Config {
65     Config() = default;
66     Config(const Config&) = delete;
67     Config(Config&&) = default;
68 
69     // All members of this struct, with the exception of |field_trials|, are
70     // expected to outlive the RTPSenderVideo object they are passed to.
71     Clock* clock = nullptr;
72     RTPSender* rtp_sender = nullptr;
73     FlexfecSender* flexfec_sender = nullptr;
74     VideoFecGenerator* fec_generator = nullptr;
75     FrameEncryptorInterface* frame_encryptor = nullptr;
76     bool require_frame_encryption = false;
77     bool enable_retransmit_all_layers = false;
78     absl::optional<int> red_payload_type;
79     const WebRtcKeyValueConfig* field_trials = nullptr;
80     rtc::scoped_refptr<FrameTransformerInterface> frame_transformer;
81     TaskQueueBase* worker_queue = nullptr;
82   };
83 
84   explicit RTPSenderVideo(const Config& config);
85 
86   virtual ~RTPSenderVideo();
87 
88   // expected_retransmission_time_ms.has_value() -> retransmission allowed.
89   // Calls to this method is assumed to be externally serialized.
90   bool SendVideo(int payload_type,
91                  absl::optional<VideoCodecType> codec_type,
92                  uint32_t rtp_timestamp,
93                  int64_t capture_time_ms,
94                  rtc::ArrayView<const uint8_t> payload,
95                  const RTPFragmentationHeader* fragmentation,
96                  RTPVideoHeader video_header,
97                  absl::optional<int64_t> expected_retransmission_time_ms);
98 
99   bool SendEncodedImage(
100       int payload_type,
101       absl::optional<VideoCodecType> codec_type,
102       uint32_t rtp_timestamp,
103       const EncodedImage& encoded_image,
104       const RTPFragmentationHeader* fragmentation,
105       RTPVideoHeader video_header,
106       absl::optional<int64_t> expected_retransmission_time_ms);
107 
108   // Configures video structures produced by encoder to send using the
109   // dependency descriptor rtp header extension. Next call to SendVideo should
110   // have video_header.frame_type == kVideoFrameKey.
111   // All calls to SendVideo after this call must use video_header compatible
112   // with the video_structure.
113   void SetVideoStructure(const FrameDependencyStructure* video_structure);
114   void SetVideoStructureUnderLock(
115       const FrameDependencyStructure* video_structure);
116 
117   // FlexFEC/ULPFEC.
118   // Set FEC rates, max frames before FEC is sent, and type of FEC masks.
119   void SetFecParameters(const FecProtectionParams& delta_params,
120                         const FecProtectionParams& key_params);
121 
122   uint32_t VideoBitrateSent() const;
123   uint32_t FecOverheadRate() const;
124 
125   // Returns the current packetization overhead rate, in bps. Note that this is
126   // the payload overhead, eg the VP8 payload headers, not the RTP headers
127   // or extension/
128   uint32_t PacketizationOverheadBps() const;
129 
130  protected:
131   static uint8_t GetTemporalId(const RTPVideoHeader& header);
132   bool AllowRetransmission(uint8_t temporal_id,
133                            int32_t retransmission_settings,
134                            int64_t expected_retransmission_time_ms);
135 
136  private:
137   struct TemporalLayerStats {
TemporalLayerStatsTemporalLayerStats138     TemporalLayerStats()
139         : frame_rate_fp1000s(kTLRateWindowSizeMs, 1000 * 1000),
140           last_frame_time_ms(0) {}
141     // Frame rate, in frames per 1000 seconds. This essentially turns the fps
142     // value into a fixed point value with three decimals. Improves precision at
143     // low frame rates.
144     RateStatistics frame_rate_fp1000s;
145     int64_t last_frame_time_ms;
146   };
147 
148   void AddRtpHeaderExtensions(
149       const RTPVideoHeader& video_header,
150       const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
151       bool first_packet,
152       bool last_packet,
153       RtpPacketToSend* packet) const
154       RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
155 
156   size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
157 
158   void LogAndSendToNetwork(
159       std::vector<std::unique_ptr<RtpPacketToSend>> packets,
160       size_t unpacketized_payload_size);
161 
red_enabled()162   bool red_enabled() const { return red_payload_type_.has_value(); }
163 
164   bool UpdateConditionalRetransmit(uint8_t temporal_id,
165                                    int64_t expected_retransmission_time_ms)
166       RTC_EXCLUSIVE_LOCKS_REQUIRED(stats_crit_);
167 
168   void MaybeUpdateCurrentPlayoutDelay(const RTPVideoHeader& header)
169       RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_);
170 
171   RTPSender* const rtp_sender_;
172   Clock* const clock_;
173 
174   const int32_t retransmission_settings_;
175 
176   // These members should only be accessed from within SendVideo() to avoid
177   // potential race conditions.
178   rtc::RaceChecker send_checker_;
179   VideoRotation last_rotation_ RTC_GUARDED_BY(send_checker_);
180   absl::optional<ColorSpace> last_color_space_ RTC_GUARDED_BY(send_checker_);
181   bool transmit_color_space_next_frame_ RTC_GUARDED_BY(send_checker_);
182   std::unique_ptr<FrameDependencyStructure> video_structure_
183       RTC_GUARDED_BY(send_checker_);
184 
185   // Current target playout delay.
186   PlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_);
187   // Flag indicating if we need to propagate |current_playout_delay_| in order
188   // to guarantee it gets delivered.
189   bool playout_delay_pending_;
190 
191   // Should never be held when calling out of this class.
192   rtc::CriticalSection crit_;
193 
194   const absl::optional<int> red_payload_type_;
195   VideoFecGenerator* const fec_generator_;
196 
197   rtc::CriticalSection stats_crit_;
198   // Bitrate used for video payload and RTP headers.
199   RateStatistics video_bitrate_ RTC_GUARDED_BY(stats_crit_);
200   RateStatistics packetization_overhead_bitrate_ RTC_GUARDED_BY(stats_crit_);
201 
202   std::map<int, TemporalLayerStats> frame_stats_by_temporal_layer_
203       RTC_GUARDED_BY(stats_crit_);
204 
205   OneTimeEvent first_frame_sent_;
206 
207   // E2EE Custom Video Frame Encryptor (optional)
208   FrameEncryptorInterface* const frame_encryptor_ = nullptr;
209   // If set to true will require all outgoing frames to pass through an
210   // initialized frame_encryptor_ before being sent out of the network.
211   // Otherwise these payloads will be dropped.
212   const bool require_frame_encryption_;
213   // Set to true if the generic descriptor should be authenticated.
214   const bool generic_descriptor_auth_experiment_;
215 
216   const bool exclude_transport_sequence_number_from_fec_experiment_;
217 
218   AbsoluteCaptureTimeSender absolute_capture_time_sender_;
219 
220   const rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate>
221       frame_transformer_delegate_;
222 };
223 
224 }  // namespace webrtc
225 
226 #endif  // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_H_
227