1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "call/rtp_video_sender.h"
12
13 #include <algorithm>
14 #include <memory>
15 #include <string>
16 #include <utility>
17
18 #include "absl/algorithm/container.h"
19 #include "absl/strings/match.h"
20 #include "api/array_view.h"
21 #include "api/transport/field_trial_based_config.h"
22 #include "api/video_codecs/video_codec.h"
23 #include "call/rtp_transport_controller_send_interface.h"
24 #include "modules/pacing/packet_router.h"
25 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
26 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
27 #include "modules/rtp_rtcp/source/rtp_sender.h"
28 #include "modules/utility/include/process_thread.h"
29 #include "modules/video_coding/include/video_codec_interface.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/location.h"
32 #include "rtc_base/logging.h"
33 #include "rtc_base/task_queue.h"
34
35 namespace webrtc {
36
37 namespace webrtc_internal_rtp_video_sender {
38
RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp,std::unique_ptr<RTPSenderVideo> sender_video,std::unique_ptr<VideoFecGenerator> fec_generator)39 RtpStreamSender::RtpStreamSender(
40 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp,
41 std::unique_ptr<RTPSenderVideo> sender_video,
42 std::unique_ptr<VideoFecGenerator> fec_generator)
43 : rtp_rtcp(std::move(rtp_rtcp)),
44 sender_video(std::move(sender_video)),
45 fec_generator(std::move(fec_generator)) {}
46
47 RtpStreamSender::~RtpStreamSender() = default;
48
49 } // namespace webrtc_internal_rtp_video_sender
50
51 namespace {
52 static const int kMinSendSidePacketHistorySize = 600;
53 // We don't do MTU discovery, so assume that we have the standard ethernet MTU.
54 static const size_t kPathMTU = 1500;
55
56 using webrtc_internal_rtp_video_sender::RtpStreamSender;
57
PayloadTypeSupportsSkippingFecPackets(const std::string & payload_name,const WebRtcKeyValueConfig & trials)58 bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name,
59 const WebRtcKeyValueConfig& trials) {
60 const VideoCodecType codecType = PayloadStringToCodecType(payload_name);
61 if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) {
62 return true;
63 }
64 if (codecType == kVideoCodecGeneric &&
65 absl::StartsWith(trials.Lookup("WebRTC-GenericPictureId"), "Enabled")) {
66 return true;
67 }
68 return false;
69 }
70
ShouldDisableRedAndUlpfec(bool flexfec_enabled,const RtpConfig & rtp_config,const WebRtcKeyValueConfig & trials)71 bool ShouldDisableRedAndUlpfec(bool flexfec_enabled,
72 const RtpConfig& rtp_config,
73 const WebRtcKeyValueConfig& trials) {
74 // Consistency of NACK and RED+ULPFEC parameters is checked in this function.
75 const bool nack_enabled = rtp_config.nack.rtp_history_ms > 0;
76
77 // Shorthands.
78 auto IsRedEnabled = [&]() { return rtp_config.ulpfec.red_payload_type >= 0; };
79 auto IsUlpfecEnabled = [&]() {
80 return rtp_config.ulpfec.ulpfec_payload_type >= 0;
81 };
82
83 bool should_disable_red_and_ulpfec = false;
84
85 if (absl::StartsWith(trials.Lookup("WebRTC-DisableUlpFecExperiment"),
86 "Enabled")) {
87 RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled.";
88 should_disable_red_and_ulpfec = true;
89 }
90
91 // If enabled, FlexFEC takes priority over RED+ULPFEC.
92 if (flexfec_enabled) {
93 if (IsUlpfecEnabled()) {
94 RTC_LOG(LS_INFO)
95 << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC.";
96 }
97 should_disable_red_and_ulpfec = true;
98 }
99
100 // Payload types without picture ID cannot determine that a stream is complete
101 // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance)
102 // is a waste of bandwidth since FEC packets still have to be transmitted.
103 // Note that this is not the case with FlexFEC.
104 if (nack_enabled && IsUlpfecEnabled() &&
105 !PayloadTypeSupportsSkippingFecPackets(rtp_config.payload_name, trials)) {
106 RTC_LOG(LS_WARNING)
107 << "Transmitting payload type without picture ID using "
108 "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets "
109 "also have to be retransmitted. Disabling ULPFEC.";
110 should_disable_red_and_ulpfec = true;
111 }
112
113 // Verify payload types.
114 if (IsUlpfecEnabled() ^ IsRedEnabled()) {
115 RTC_LOG(LS_WARNING)
116 << "Only RED or only ULPFEC enabled, but not both. Disabling both.";
117 should_disable_red_and_ulpfec = true;
118 }
119
120 return should_disable_red_and_ulpfec;
121 }
122
123 // TODO(brandtr): Update this function when we support multistream protection.
MaybeCreateFecGenerator(Clock * clock,const RtpConfig & rtp,const std::map<uint32_t,RtpState> & suspended_ssrcs,int simulcast_index,const WebRtcKeyValueConfig & trials)124 std::unique_ptr<VideoFecGenerator> MaybeCreateFecGenerator(
125 Clock* clock,
126 const RtpConfig& rtp,
127 const std::map<uint32_t, RtpState>& suspended_ssrcs,
128 int simulcast_index,
129 const WebRtcKeyValueConfig& trials) {
130 // If flexfec is configured that takes priority.
131 if (rtp.flexfec.payload_type >= 0) {
132 RTC_DCHECK_GE(rtp.flexfec.payload_type, 0);
133 RTC_DCHECK_LE(rtp.flexfec.payload_type, 127);
134 if (rtp.flexfec.ssrc == 0) {
135 RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
136 "Therefore disabling FlexFEC.";
137 return nullptr;
138 }
139 if (rtp.flexfec.protected_media_ssrcs.empty()) {
140 RTC_LOG(LS_WARNING)
141 << "FlexFEC is enabled, but no protected media SSRC given. "
142 "Therefore disabling FlexFEC.";
143 return nullptr;
144 }
145
146 if (rtp.flexfec.protected_media_ssrcs.size() > 1) {
147 RTC_LOG(LS_WARNING)
148 << "The supplied FlexfecConfig contained multiple protected "
149 "media streams, but our implementation currently only "
150 "supports protecting a single media stream. "
151 "To avoid confusion, disabling FlexFEC completely.";
152 return nullptr;
153 }
154
155 if (absl::c_find(rtp.flexfec.protected_media_ssrcs,
156 rtp.ssrcs[simulcast_index]) ==
157 rtp.flexfec.protected_media_ssrcs.end()) {
158 // Media SSRC not among flexfec protected SSRCs.
159 return nullptr;
160 }
161
162 const RtpState* rtp_state = nullptr;
163 auto it = suspended_ssrcs.find(rtp.flexfec.ssrc);
164 if (it != suspended_ssrcs.end()) {
165 rtp_state = &it->second;
166 }
167
168 RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size());
169 return std::make_unique<FlexfecSender>(
170 rtp.flexfec.payload_type, rtp.flexfec.ssrc,
171 rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions,
172 RTPSender::FecExtensionSizes(), rtp_state, clock);
173 } else if (rtp.ulpfec.red_payload_type >= 0 &&
174 rtp.ulpfec.ulpfec_payload_type >= 0 &&
175 !ShouldDisableRedAndUlpfec(/*flexfec_enabled=*/false, rtp,
176 trials)) {
177 // Flexfec not configured, but ulpfec is and is not disabled.
178 return std::make_unique<UlpfecGenerator>(
179 rtp.ulpfec.red_payload_type, rtp.ulpfec.ulpfec_payload_type, clock);
180 }
181
182 // Not a single FEC is given.
183 return nullptr;
184 }
185
CreateRtpStreamSenders(Clock * clock,const RtpConfig & rtp_config,const RtpSenderObservers & observers,int rtcp_report_interval_ms,Transport * send_transport,RtcpBandwidthObserver * bandwidth_callback,RtpTransportControllerSendInterface * transport,const std::map<uint32_t,RtpState> & suspended_ssrcs,RtcEventLog * event_log,RateLimiter * retransmission_rate_limiter,FrameEncryptorInterface * frame_encryptor,const CryptoOptions & crypto_options,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,const WebRtcKeyValueConfig & trials)186 std::vector<RtpStreamSender> CreateRtpStreamSenders(
187 Clock* clock,
188 const RtpConfig& rtp_config,
189 const RtpSenderObservers& observers,
190 int rtcp_report_interval_ms,
191 Transport* send_transport,
192 RtcpBandwidthObserver* bandwidth_callback,
193 RtpTransportControllerSendInterface* transport,
194 const std::map<uint32_t, RtpState>& suspended_ssrcs,
195 RtcEventLog* event_log,
196 RateLimiter* retransmission_rate_limiter,
197 FrameEncryptorInterface* frame_encryptor,
198 const CryptoOptions& crypto_options,
199 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
200 const WebRtcKeyValueConfig& trials) {
201 RTC_DCHECK_GT(rtp_config.ssrcs.size(), 0);
202
203 RtpRtcpInterface::Configuration configuration;
204 configuration.clock = clock;
205 configuration.audio = false;
206 configuration.receiver_only = false;
207 configuration.outgoing_transport = send_transport;
208 configuration.intra_frame_callback = observers.intra_frame_callback;
209 configuration.rtcp_loss_notification_observer =
210 observers.rtcp_loss_notification_observer;
211 configuration.bandwidth_callback = bandwidth_callback;
212 configuration.network_state_estimate_observer =
213 transport->network_state_estimate_observer();
214 configuration.transport_feedback_callback =
215 transport->transport_feedback_observer();
216 configuration.rtt_stats = observers.rtcp_rtt_stats;
217 configuration.rtcp_packet_type_counter_observer =
218 observers.rtcp_type_observer;
219 configuration.rtcp_statistics_callback = observers.rtcp_stats;
220 configuration.report_block_data_observer =
221 observers.report_block_data_observer;
222 configuration.paced_sender = transport->packet_sender();
223 configuration.send_bitrate_observer = observers.bitrate_observer;
224 configuration.send_side_delay_observer = observers.send_delay_observer;
225 configuration.send_packet_observer = observers.send_packet_observer;
226 configuration.event_log = event_log;
227 configuration.retransmission_rate_limiter = retransmission_rate_limiter;
228 configuration.rtp_stats_callback = observers.rtp_stats;
229 configuration.frame_encryptor = frame_encryptor;
230 configuration.require_frame_encryption =
231 crypto_options.sframe.require_frame_encryption;
232 configuration.extmap_allow_mixed = rtp_config.extmap_allow_mixed;
233 configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;
234 configuration.field_trials = &trials;
235
236 std::vector<RtpStreamSender> rtp_streams;
237
238 RTC_DCHECK(rtp_config.rtx.ssrcs.empty() ||
239 rtp_config.rtx.ssrcs.size() == rtp_config.ssrcs.size());
240 for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) {
241 RTPSenderVideo::Config video_config;
242 configuration.local_media_ssrc = rtp_config.ssrcs[i];
243
244 std::unique_ptr<VideoFecGenerator> fec_generator =
245 MaybeCreateFecGenerator(clock, rtp_config, suspended_ssrcs, i, trials);
246 configuration.fec_generator = fec_generator.get();
247
248 configuration.rtx_send_ssrc =
249 rtp_config.GetRtxSsrcAssociatedWithMediaSsrc(rtp_config.ssrcs[i]);
250 RTC_DCHECK_EQ(configuration.rtx_send_ssrc.has_value(),
251 !rtp_config.rtx.ssrcs.empty());
252
253 configuration.need_rtp_packet_infos = rtp_config.lntf.enabled;
254
255 std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp(
256 ModuleRtpRtcpImpl2::Create(configuration));
257 rtp_rtcp->SetSendingStatus(false);
258 rtp_rtcp->SetSendingMediaStatus(false);
259 rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
260 // Set NACK.
261 rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize);
262
263 video_config.clock = configuration.clock;
264 video_config.rtp_sender = rtp_rtcp->RtpSender();
265 video_config.frame_encryptor = frame_encryptor;
266 video_config.require_frame_encryption =
267 crypto_options.sframe.require_frame_encryption;
268 video_config.enable_retransmit_all_layers = false;
269 video_config.field_trials = &trials;
270
271 const bool using_flexfec =
272 fec_generator &&
273 fec_generator->GetFecType() == VideoFecGenerator::FecType::kFlexFec;
274 const bool should_disable_red_and_ulpfec =
275 ShouldDisableRedAndUlpfec(using_flexfec, rtp_config, trials);
276 if (!should_disable_red_and_ulpfec &&
277 rtp_config.ulpfec.red_payload_type != -1) {
278 video_config.red_payload_type = rtp_config.ulpfec.red_payload_type;
279 }
280 if (fec_generator) {
281 video_config.fec_type = fec_generator->GetFecType();
282 video_config.fec_overhead_bytes = fec_generator->MaxPacketOverhead();
283 }
284 video_config.frame_transformer = frame_transformer;
285 video_config.send_transport_queue = transport->GetWorkerQueue()->Get();
286 auto sender_video = std::make_unique<RTPSenderVideo>(video_config);
287 rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video),
288 std::move(fec_generator));
289 }
290 return rtp_streams;
291 }
292
CalculateOverheadRate(DataRate data_rate,DataSize packet_size,DataSize overhead_per_packet)293 DataRate CalculateOverheadRate(DataRate data_rate,
294 DataSize packet_size,
295 DataSize overhead_per_packet) {
296 Frequency packet_rate = data_rate / packet_size;
297 // TOSO(srte): We should not need to round to nearest whole packet per second
298 // rate here.
299 return packet_rate.RoundUpTo(Frequency::Hertz(1)) * overhead_per_packet;
300 }
301
GetVideoCodecType(const RtpConfig & config)302 absl::optional<VideoCodecType> GetVideoCodecType(const RtpConfig& config) {
303 if (config.raw_payload) {
304 return absl::nullopt;
305 }
306 return PayloadStringToCodecType(config.payload_name);
307 }
TransportSeqNumExtensionConfigured(const RtpConfig & config)308 bool TransportSeqNumExtensionConfigured(const RtpConfig& config) {
309 return absl::c_any_of(config.extensions, [](const RtpExtension& ext) {
310 return ext.uri == RtpExtension::kTransportSequenceNumberUri;
311 });
312 }
313 } // namespace
314
RtpVideoSender(Clock * clock,std::map<uint32_t,RtpState> suspended_ssrcs,const std::map<uint32_t,RtpPayloadState> & states,const RtpConfig & rtp_config,int rtcp_report_interval_ms,Transport * send_transport,const RtpSenderObservers & observers,RtpTransportControllerSendInterface * transport,RtcEventLog * event_log,RateLimiter * retransmission_limiter,std::unique_ptr<FecController> fec_controller,FrameEncryptorInterface * frame_encryptor,const CryptoOptions & crypto_options,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)315 RtpVideoSender::RtpVideoSender(
316 Clock* clock,
317 std::map<uint32_t, RtpState> suspended_ssrcs,
318 const std::map<uint32_t, RtpPayloadState>& states,
319 const RtpConfig& rtp_config,
320 int rtcp_report_interval_ms,
321 Transport* send_transport,
322 const RtpSenderObservers& observers,
323 RtpTransportControllerSendInterface* transport,
324 RtcEventLog* event_log,
325 RateLimiter* retransmission_limiter,
326 std::unique_ptr<FecController> fec_controller,
327 FrameEncryptorInterface* frame_encryptor,
328 const CryptoOptions& crypto_options,
329 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
330 : send_side_bwe_with_overhead_(!absl::StartsWith(
331 field_trials_.Lookup("WebRTC-SendSideBwe-WithOverhead"),
332 "Disabled")),
333 has_packet_feedback_(TransportSeqNumExtensionConfigured(rtp_config)),
334 active_(false),
335 module_process_thread_(nullptr),
336 suspended_ssrcs_(std::move(suspended_ssrcs)),
337 fec_controller_(std::move(fec_controller)),
338 fec_allowed_(true),
339 rtp_streams_(CreateRtpStreamSenders(clock,
340 rtp_config,
341 observers,
342 rtcp_report_interval_ms,
343 send_transport,
344 transport->GetBandwidthObserver(),
345 transport,
346 suspended_ssrcs_,
347 event_log,
348 retransmission_limiter,
349 frame_encryptor,
350 crypto_options,
351 std::move(frame_transformer),
352 field_trials_)),
353 rtp_config_(rtp_config),
354 codec_type_(GetVideoCodecType(rtp_config)),
355 transport_(transport),
356 transport_overhead_bytes_per_packet_(0),
357 encoder_target_rate_bps_(0),
358 frame_counts_(rtp_config.ssrcs.size()),
359 frame_count_observer_(observers.frame_count_observer) {
360 RTC_DCHECK_EQ(rtp_config_.ssrcs.size(), rtp_streams_.size());
361 if (send_side_bwe_with_overhead_ && has_packet_feedback_)
362 transport_->IncludeOverheadInPacedSender();
363 module_process_thread_checker_.Detach();
364 // SSRCs are assumed to be sorted in the same order as |rtp_modules|.
365 for (uint32_t ssrc : rtp_config_.ssrcs) {
366 // Restore state if it previously existed.
367 const RtpPayloadState* state = nullptr;
368 auto it = states.find(ssrc);
369 if (it != states.end()) {
370 state = &it->second;
371 shared_frame_id_ = std::max(shared_frame_id_, state->shared_frame_id);
372 }
373 params_.push_back(RtpPayloadParams(ssrc, state, field_trials_));
374 }
375
376 // RTP/RTCP initialization.
377
378 // We add the highest spatial layer first to ensure it'll be prioritized
379 // when sending padding, with the hope that the packet rate will be smaller,
380 // and that it's more important to protect than the lower layers.
381
382 // TODO(nisse): Consider moving registration with PacketRouter last, after the
383 // modules are fully configured.
384 for (const RtpStreamSender& stream : rtp_streams_) {
385 constexpr bool remb_candidate = true;
386 transport->packet_router()->AddSendRtpModule(stream.rtp_rtcp.get(),
387 remb_candidate);
388 }
389
390 for (size_t i = 0; i < rtp_config_.extensions.size(); ++i) {
391 const std::string& extension = rtp_config_.extensions[i].uri;
392 int id = rtp_config_.extensions[i].id;
393 RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
394 for (const RtpStreamSender& stream : rtp_streams_) {
395 stream.rtp_rtcp->RegisterRtpHeaderExtension(extension, id);
396 }
397 }
398
399 ConfigureSsrcs();
400 ConfigureRids();
401
402 if (!rtp_config_.mid.empty()) {
403 for (const RtpStreamSender& stream : rtp_streams_) {
404 stream.rtp_rtcp->SetMid(rtp_config_.mid);
405 }
406 }
407
408 bool fec_enabled = false;
409 for (const RtpStreamSender& stream : rtp_streams_) {
410 // Simulcast has one module for each layer. Set the CNAME on all modules.
411 stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str());
412 stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size);
413 stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type,
414 kVideoPayloadTypeFrequency);
415 if (stream.fec_generator != nullptr) {
416 fec_enabled = true;
417 }
418 }
419 // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
420 // so enable that logic if either of those FEC schemes are enabled.
421 fec_controller_->SetProtectionMethod(fec_enabled, NackEnabled());
422
423 fec_controller_->SetProtectionCallback(this);
424 // Signal congestion controller this object is ready for OnPacket* callbacks.
425 transport_->GetStreamFeedbackProvider()->RegisterStreamFeedbackObserver(
426 rtp_config_.ssrcs, this);
427 }
428
~RtpVideoSender()429 RtpVideoSender::~RtpVideoSender() {
430 for (const RtpStreamSender& stream : rtp_streams_) {
431 transport_->packet_router()->RemoveSendRtpModule(stream.rtp_rtcp.get());
432 }
433 transport_->GetStreamFeedbackProvider()->DeRegisterStreamFeedbackObserver(
434 this);
435 }
436
RegisterProcessThread(ProcessThread * module_process_thread)437 void RtpVideoSender::RegisterProcessThread(
438 ProcessThread* module_process_thread) {
439 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
440 RTC_DCHECK(!module_process_thread_);
441 module_process_thread_ = module_process_thread;
442
443 for (const RtpStreamSender& stream : rtp_streams_) {
444 module_process_thread_->RegisterModule(stream.rtp_rtcp.get(),
445 RTC_FROM_HERE);
446 }
447 }
448
DeRegisterProcessThread()449 void RtpVideoSender::DeRegisterProcessThread() {
450 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
451 for (const RtpStreamSender& stream : rtp_streams_)
452 module_process_thread_->DeRegisterModule(stream.rtp_rtcp.get());
453 }
454
SetActive(bool active)455 void RtpVideoSender::SetActive(bool active) {
456 MutexLock lock(&mutex_);
457 if (active_ == active)
458 return;
459 const std::vector<bool> active_modules(rtp_streams_.size(), active);
460 SetActiveModulesLocked(active_modules);
461 }
462
SetActiveModules(const std::vector<bool> active_modules)463 void RtpVideoSender::SetActiveModules(const std::vector<bool> active_modules) {
464 MutexLock lock(&mutex_);
465 return SetActiveModulesLocked(active_modules);
466 }
467
SetActiveModulesLocked(const std::vector<bool> active_modules)468 void RtpVideoSender::SetActiveModulesLocked(
469 const std::vector<bool> active_modules) {
470 RTC_DCHECK_EQ(rtp_streams_.size(), active_modules.size());
471 active_ = false;
472 for (size_t i = 0; i < active_modules.size(); ++i) {
473 if (active_modules[i]) {
474 active_ = true;
475 }
476 // Sends a kRtcpByeCode when going from true to false.
477 rtp_streams_[i].rtp_rtcp->SetSendingStatus(active_modules[i]);
478 // If set to false this module won't send media.
479 rtp_streams_[i].rtp_rtcp->SetSendingMediaStatus(active_modules[i]);
480 }
481 }
482
IsActive()483 bool RtpVideoSender::IsActive() {
484 MutexLock lock(&mutex_);
485 return IsActiveLocked();
486 }
487
IsActiveLocked()488 bool RtpVideoSender::IsActiveLocked() {
489 return active_ && !rtp_streams_.empty();
490 }
491
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info)492 EncodedImageCallback::Result RtpVideoSender::OnEncodedImage(
493 const EncodedImage& encoded_image,
494 const CodecSpecificInfo* codec_specific_info) {
495 fec_controller_->UpdateWithEncodedData(encoded_image.size(),
496 encoded_image._frameType);
497 MutexLock lock(&mutex_);
498 RTC_DCHECK(!rtp_streams_.empty());
499 if (!active_)
500 return Result(Result::ERROR_SEND_FAILED);
501
502 shared_frame_id_++;
503 size_t stream_index = 0;
504 if (codec_specific_info &&
505 (codec_specific_info->codecType == kVideoCodecVP8 ||
506 codec_specific_info->codecType == kVideoCodecH264 ||
507 codec_specific_info->codecType == kVideoCodecGeneric)) {
508 // Map spatial index to simulcast.
509 stream_index = encoded_image.SpatialIndex().value_or(0);
510 }
511 RTC_DCHECK_LT(stream_index, rtp_streams_.size());
512
513 uint32_t rtp_timestamp =
514 encoded_image.Timestamp() +
515 rtp_streams_[stream_index].rtp_rtcp->StartTimestamp();
516
517 // RTCPSender has it's own copy of the timestamp offset, added in
518 // RTCPSender::BuildSR, hence we must not add the in the offset for this call.
519 // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine
520 // knowledge of the offset to a single place.
521 if (!rtp_streams_[stream_index].rtp_rtcp->OnSendingRtpFrame(
522 encoded_image.Timestamp(), encoded_image.capture_time_ms_,
523 rtp_config_.payload_type,
524 encoded_image._frameType == VideoFrameType::kVideoFrameKey)) {
525 // The payload router could be active but this module isn't sending.
526 return Result(Result::ERROR_SEND_FAILED);
527 }
528
529 absl::optional<int64_t> expected_retransmission_time_ms;
530 if (encoded_image.RetransmissionAllowed()) {
531 expected_retransmission_time_ms =
532 rtp_streams_[stream_index].rtp_rtcp->ExpectedRetransmissionTimeMs();
533 }
534
535 if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
536 // If encoder adapter produce FrameDependencyStructure, pass it so that
537 // dependency descriptor rtp header extension can be used.
538 // If not supported, disable using dependency descriptor by passing nullptr.
539 rtp_streams_[stream_index].sender_video->SetVideoStructure(
540 (codec_specific_info && codec_specific_info->template_structure)
541 ? &*codec_specific_info->template_structure
542 : nullptr);
543 }
544
545 bool send_result = rtp_streams_[stream_index].sender_video->SendEncodedImage(
546 rtp_config_.payload_type, codec_type_, rtp_timestamp, encoded_image,
547 params_[stream_index].GetRtpVideoHeader(
548 encoded_image, codec_specific_info, shared_frame_id_),
549 expected_retransmission_time_ms);
550 if (frame_count_observer_) {
551 FrameCounts& counts = frame_counts_[stream_index];
552 if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
553 ++counts.key_frames;
554 } else if (encoded_image._frameType == VideoFrameType::kVideoFrameDelta) {
555 ++counts.delta_frames;
556 } else {
557 RTC_DCHECK(encoded_image._frameType == VideoFrameType::kEmptyFrame);
558 }
559 frame_count_observer_->FrameCountUpdated(counts,
560 rtp_config_.ssrcs[stream_index]);
561 }
562 if (!send_result)
563 return Result(Result::ERROR_SEND_FAILED);
564
565 return Result(Result::OK, rtp_timestamp);
566 }
567
OnBitrateAllocationUpdated(const VideoBitrateAllocation & bitrate)568 void RtpVideoSender::OnBitrateAllocationUpdated(
569 const VideoBitrateAllocation& bitrate) {
570 MutexLock lock(&mutex_);
571 if (IsActiveLocked()) {
572 if (rtp_streams_.size() == 1) {
573 // If spatial scalability is enabled, it is covered by a single stream.
574 rtp_streams_[0].rtp_rtcp->SetVideoBitrateAllocation(bitrate);
575 } else {
576 std::vector<absl::optional<VideoBitrateAllocation>> layer_bitrates =
577 bitrate.GetSimulcastAllocations();
578 // Simulcast is in use, split the VideoBitrateAllocation into one struct
579 // per rtp stream, moving over the temporal layer allocation.
580 for (size_t i = 0; i < rtp_streams_.size(); ++i) {
581 // The next spatial layer could be used if the current one is
582 // inactive.
583 if (layer_bitrates[i]) {
584 rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation(
585 *layer_bitrates[i]);
586 } else {
587 // Signal a 0 bitrate on a simulcast stream.
588 rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation(
589 VideoBitrateAllocation());
590 }
591 }
592 }
593 }
594 }
OnVideoLayersAllocationUpdated(const VideoLayersAllocation & allocation)595 void RtpVideoSender::OnVideoLayersAllocationUpdated(
596 const VideoLayersAllocation& allocation) {
597 MutexLock lock(&mutex_);
598 if (IsActiveLocked()) {
599 for (size_t i = 0; i < rtp_streams_.size(); ++i) {
600 VideoLayersAllocation stream_allocation = allocation;
601 stream_allocation.rtp_stream_index = i;
602 rtp_streams_[i].sender_video->SetVideoLayersAllocation(
603 std::move(stream_allocation));
604 }
605 }
606 }
607
NackEnabled() const608 bool RtpVideoSender::NackEnabled() const {
609 const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0;
610 return nack_enabled;
611 }
612
GetPacketizationOverheadRate() const613 uint32_t RtpVideoSender::GetPacketizationOverheadRate() const {
614 uint32_t packetization_overhead_bps = 0;
615 for (size_t i = 0; i < rtp_streams_.size(); ++i) {
616 if (rtp_streams_[i].rtp_rtcp->SendingMedia()) {
617 packetization_overhead_bps +=
618 rtp_streams_[i].sender_video->PacketizationOverheadBps();
619 }
620 }
621 return packetization_overhead_bps;
622 }
623
DeliverRtcp(const uint8_t * packet,size_t length)624 void RtpVideoSender::DeliverRtcp(const uint8_t* packet, size_t length) {
625 // Runs on a network thread.
626 for (const RtpStreamSender& stream : rtp_streams_)
627 stream.rtp_rtcp->IncomingRtcpPacket(packet, length);
628 }
629
ConfigureSsrcs()630 void RtpVideoSender::ConfigureSsrcs() {
631 // Configure regular SSRCs.
632 RTC_CHECK(ssrc_to_rtp_module_.empty());
633 for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) {
634 uint32_t ssrc = rtp_config_.ssrcs[i];
635 RtpRtcpInterface* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get();
636
637 // Restore RTP state if previous existed.
638 auto it = suspended_ssrcs_.find(ssrc);
639 if (it != suspended_ssrcs_.end())
640 rtp_rtcp->SetRtpState(it->second);
641
642 ssrc_to_rtp_module_[ssrc] = rtp_rtcp;
643 }
644
645 // Set up RTX if available.
646 if (rtp_config_.rtx.ssrcs.empty())
647 return;
648
649 RTC_DCHECK_EQ(rtp_config_.rtx.ssrcs.size(), rtp_config_.ssrcs.size());
650 for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) {
651 uint32_t ssrc = rtp_config_.rtx.ssrcs[i];
652 RtpRtcpInterface* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get();
653 auto it = suspended_ssrcs_.find(ssrc);
654 if (it != suspended_ssrcs_.end())
655 rtp_rtcp->SetRtxState(it->second);
656 }
657
658 // Configure RTX payload types.
659 RTC_DCHECK_GE(rtp_config_.rtx.payload_type, 0);
660 for (const RtpStreamSender& stream : rtp_streams_) {
661 stream.rtp_rtcp->SetRtxSendPayloadType(rtp_config_.rtx.payload_type,
662 rtp_config_.payload_type);
663 stream.rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted |
664 kRtxRedundantPayloads);
665 }
666 if (rtp_config_.ulpfec.red_payload_type != -1 &&
667 rtp_config_.ulpfec.red_rtx_payload_type != -1) {
668 for (const RtpStreamSender& stream : rtp_streams_) {
669 stream.rtp_rtcp->SetRtxSendPayloadType(
670 rtp_config_.ulpfec.red_rtx_payload_type,
671 rtp_config_.ulpfec.red_payload_type);
672 }
673 }
674 }
675
ConfigureRids()676 void RtpVideoSender::ConfigureRids() {
677 if (rtp_config_.rids.empty())
678 return;
679
680 // Some streams could have been disabled, but the rids are still there.
681 // This will occur when simulcast has been disabled for a codec (e.g. VP9)
682 RTC_DCHECK(rtp_config_.rids.size() >= rtp_streams_.size());
683 for (size_t i = 0; i < rtp_streams_.size(); ++i) {
684 rtp_streams_[i].rtp_rtcp->SetRid(rtp_config_.rids[i]);
685 }
686 }
687
OnNetworkAvailability(bool network_available)688 void RtpVideoSender::OnNetworkAvailability(bool network_available) {
689 for (const RtpStreamSender& stream : rtp_streams_) {
690 stream.rtp_rtcp->SetRTCPStatus(network_available ? rtp_config_.rtcp_mode
691 : RtcpMode::kOff);
692 }
693 }
694
GetRtpStates() const695 std::map<uint32_t, RtpState> RtpVideoSender::GetRtpStates() const {
696 std::map<uint32_t, RtpState> rtp_states;
697
698 for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) {
699 uint32_t ssrc = rtp_config_.ssrcs[i];
700 RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC());
701 rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState();
702
703 // Only happens during shutdown, when RTP module is already inactive,
704 // so OK to call fec generator here.
705 if (rtp_streams_[i].fec_generator) {
706 absl::optional<RtpState> fec_state =
707 rtp_streams_[i].fec_generator->GetRtpState();
708 if (fec_state) {
709 uint32_t ssrc = rtp_config_.flexfec.ssrc;
710 rtp_states[ssrc] = *fec_state;
711 }
712 }
713 }
714
715 for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) {
716 uint32_t ssrc = rtp_config_.rtx.ssrcs[i];
717 rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState();
718 }
719
720 return rtp_states;
721 }
722
GetRtpPayloadStates() const723 std::map<uint32_t, RtpPayloadState> RtpVideoSender::GetRtpPayloadStates()
724 const {
725 MutexLock lock(&mutex_);
726 std::map<uint32_t, RtpPayloadState> payload_states;
727 for (const auto& param : params_) {
728 payload_states[param.ssrc()] = param.state();
729 payload_states[param.ssrc()].shared_frame_id = shared_frame_id_;
730 }
731 return payload_states;
732 }
733
OnTransportOverheadChanged(size_t transport_overhead_bytes_per_packet)734 void RtpVideoSender::OnTransportOverheadChanged(
735 size_t transport_overhead_bytes_per_packet) {
736 MutexLock lock(&mutex_);
737 transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet;
738
739 size_t max_rtp_packet_size =
740 std::min(rtp_config_.max_packet_size,
741 kPathMTU - transport_overhead_bytes_per_packet_);
742 for (const RtpStreamSender& stream : rtp_streams_) {
743 stream.rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size);
744 }
745 }
746
OnBitrateUpdated(BitrateAllocationUpdate update,int framerate)747 void RtpVideoSender::OnBitrateUpdated(BitrateAllocationUpdate update,
748 int framerate) {
749 // Substract overhead from bitrate.
750 MutexLock lock(&mutex_);
751 size_t num_active_streams = 0;
752 size_t overhead_bytes_per_packet = 0;
753 for (const auto& stream : rtp_streams_) {
754 if (stream.rtp_rtcp->SendingMedia()) {
755 overhead_bytes_per_packet += stream.rtp_rtcp->ExpectedPerPacketOverhead();
756 ++num_active_streams;
757 }
758 }
759 if (num_active_streams > 1) {
760 overhead_bytes_per_packet /= num_active_streams;
761 }
762
763 DataSize packet_overhead = DataSize::Bytes(
764 overhead_bytes_per_packet + transport_overhead_bytes_per_packet_);
765 DataSize max_total_packet_size = DataSize::Bytes(
766 rtp_config_.max_packet_size + transport_overhead_bytes_per_packet_);
767 uint32_t payload_bitrate_bps = update.target_bitrate.bps();
768 if (send_side_bwe_with_overhead_ && has_packet_feedback_) {
769 DataRate overhead_rate = CalculateOverheadRate(
770 update.target_bitrate, max_total_packet_size, packet_overhead);
771 // TODO(srte): We probably should not accept 0 payload bitrate here.
772 payload_bitrate_bps = rtc::saturated_cast<uint32_t>(payload_bitrate_bps -
773 overhead_rate.bps());
774 }
775
776 // Get the encoder target rate. It is the estimated network rate -
777 // protection overhead.
778 // TODO(srte): We should multiply with 255 here.
779 encoder_target_rate_bps_ = fec_controller_->UpdateFecRates(
780 payload_bitrate_bps, framerate,
781 rtc::saturated_cast<uint8_t>(update.packet_loss_ratio * 256),
782 loss_mask_vector_, update.round_trip_time.ms());
783 if (!fec_allowed_) {
784 encoder_target_rate_bps_ = payload_bitrate_bps;
785 // fec_controller_->UpdateFecRates() was still called so as to allow
786 // |fec_controller_| to update whatever internal state it might have,
787 // since |fec_allowed_| may be toggled back on at any moment.
788 }
789
790 // Subtract packetization overhead from the encoder target. If target rate
791 // is really low, cap the overhead at 50%. This also avoids the case where
792 // |encoder_target_rate_bps_| is 0 due to encoder pause event while the
793 // packetization rate is positive since packets are still flowing.
794 uint32_t packetization_rate_bps =
795 std::min(GetPacketizationOverheadRate(), encoder_target_rate_bps_ / 2);
796 encoder_target_rate_bps_ -= packetization_rate_bps;
797
798 loss_mask_vector_.clear();
799
800 uint32_t encoder_overhead_rate_bps = 0;
801 if (send_side_bwe_with_overhead_ && has_packet_feedback_) {
802 // TODO(srte): The packet size should probably be the same as in the
803 // CalculateOverheadRate call above (just max_total_packet_size), it doesn't
804 // make sense to use different packet rates for different overhead
805 // calculations.
806 DataRate encoder_overhead_rate = CalculateOverheadRate(
807 DataRate::BitsPerSec(encoder_target_rate_bps_),
808 max_total_packet_size - DataSize::Bytes(overhead_bytes_per_packet),
809 packet_overhead);
810 encoder_overhead_rate_bps = std::min(
811 encoder_overhead_rate.bps<uint32_t>(),
812 update.target_bitrate.bps<uint32_t>() - encoder_target_rate_bps_);
813 }
814 // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled
815 // protection_bitrate includes overhead.
816 const uint32_t media_rate = encoder_target_rate_bps_ +
817 encoder_overhead_rate_bps +
818 packetization_rate_bps;
819 RTC_DCHECK_GE(update.target_bitrate, DataRate::BitsPerSec(media_rate));
820 protection_bitrate_bps_ = update.target_bitrate.bps() - media_rate;
821 }
822
GetPayloadBitrateBps() const823 uint32_t RtpVideoSender::GetPayloadBitrateBps() const {
824 return encoder_target_rate_bps_;
825 }
826
GetProtectionBitrateBps() const827 uint32_t RtpVideoSender::GetProtectionBitrateBps() const {
828 return protection_bitrate_bps_;
829 }
830
GetSentRtpPacketInfos(uint32_t ssrc,rtc::ArrayView<const uint16_t> sequence_numbers) const831 std::vector<RtpSequenceNumberMap::Info> RtpVideoSender::GetSentRtpPacketInfos(
832 uint32_t ssrc,
833 rtc::ArrayView<const uint16_t> sequence_numbers) const {
834 for (const auto& rtp_stream : rtp_streams_) {
835 if (ssrc == rtp_stream.rtp_rtcp->SSRC()) {
836 return rtp_stream.rtp_rtcp->GetSentRtpPacketInfos(sequence_numbers);
837 }
838 }
839 return std::vector<RtpSequenceNumberMap::Info>();
840 }
841
ProtectionRequest(const FecProtectionParams * delta_params,const FecProtectionParams * key_params,uint32_t * sent_video_rate_bps,uint32_t * sent_nack_rate_bps,uint32_t * sent_fec_rate_bps)842 int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params,
843 const FecProtectionParams* key_params,
844 uint32_t* sent_video_rate_bps,
845 uint32_t* sent_nack_rate_bps,
846 uint32_t* sent_fec_rate_bps) {
847 *sent_video_rate_bps = 0;
848 *sent_nack_rate_bps = 0;
849 *sent_fec_rate_bps = 0;
850 for (const RtpStreamSender& stream : rtp_streams_) {
851 stream.rtp_rtcp->SetFecProtectionParams(*delta_params, *key_params);
852
853 auto send_bitrate = stream.rtp_rtcp->GetSendRates();
854 *sent_video_rate_bps += send_bitrate[RtpPacketMediaType::kVideo].bps();
855 *sent_fec_rate_bps +=
856 send_bitrate[RtpPacketMediaType::kForwardErrorCorrection].bps();
857 *sent_nack_rate_bps +=
858 send_bitrate[RtpPacketMediaType::kRetransmission].bps();
859 }
860 return 0;
861 }
862
SetFecAllowed(bool fec_allowed)863 void RtpVideoSender::SetFecAllowed(bool fec_allowed) {
864 MutexLock lock(&mutex_);
865 fec_allowed_ = fec_allowed;
866 }
867
OnPacketFeedbackVector(std::vector<StreamPacketInfo> packet_feedback_vector)868 void RtpVideoSender::OnPacketFeedbackVector(
869 std::vector<StreamPacketInfo> packet_feedback_vector) {
870 if (fec_controller_->UseLossVectorMask()) {
871 MutexLock lock(&mutex_);
872 for (const StreamPacketInfo& packet : packet_feedback_vector) {
873 loss_mask_vector_.push_back(!packet.received);
874 }
875 }
876
877 // Map from SSRC to all acked packets for that RTP module.
878 std::map<uint32_t, std::vector<uint16_t>> acked_packets_per_ssrc;
879 for (const StreamPacketInfo& packet : packet_feedback_vector) {
880 if (packet.received) {
881 acked_packets_per_ssrc[packet.ssrc].push_back(packet.rtp_sequence_number);
882 }
883 }
884
885 // Map from SSRC to vector of RTP sequence numbers that are indicated as
886 // lost by feedback, without being trailed by any received packets.
887 std::map<uint32_t, std::vector<uint16_t>> early_loss_detected_per_ssrc;
888
889 for (const StreamPacketInfo& packet : packet_feedback_vector) {
890 if (!packet.received) {
891 // Last known lost packet, might not be detectable as lost by remote
892 // jitter buffer.
893 early_loss_detected_per_ssrc[packet.ssrc].push_back(
894 packet.rtp_sequence_number);
895 } else {
896 // Packet received, so any loss prior to this is already detectable.
897 early_loss_detected_per_ssrc.erase(packet.ssrc);
898 }
899 }
900
901 for (const auto& kv : early_loss_detected_per_ssrc) {
902 const uint32_t ssrc = kv.first;
903 auto it = ssrc_to_rtp_module_.find(ssrc);
904 RTC_DCHECK(it != ssrc_to_rtp_module_.end());
905 RTPSender* rtp_sender = it->second->RtpSender();
906 for (uint16_t sequence_number : kv.second) {
907 rtp_sender->ReSendPacket(sequence_number);
908 }
909 }
910
911 for (const auto& kv : acked_packets_per_ssrc) {
912 const uint32_t ssrc = kv.first;
913 auto it = ssrc_to_rtp_module_.find(ssrc);
914 if (it == ssrc_to_rtp_module_.end()) {
915 // Packets not for a media SSRC, so likely RTX or FEC. If so, ignore
916 // since there's no RTP history to clean up anyway.
917 continue;
918 }
919 rtc::ArrayView<const uint16_t> rtp_sequence_numbers(kv.second);
920 it->second->OnPacketsAcknowledged(rtp_sequence_numbers);
921 }
922 }
923
SetEncodingData(size_t width,size_t height,size_t num_temporal_layers)924 void RtpVideoSender::SetEncodingData(size_t width,
925 size_t height,
926 size_t num_temporal_layers) {
927 fec_controller_->SetEncodingData(width, height, num_temporal_layers,
928 rtp_config_.max_packet_size);
929 }
930 } // namespace webrtc
931