1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/rtp_rtcp/source/rtp_sender.h"
12 
13 #include <algorithm>
14 #include <utility>
15 
16 #include "logging/rtc_event_log/events/rtc_event_rtp_packet_outgoing.h"
17 #include "logging/rtc_event_log/rtc_event_log.h"
18 #include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
19 #include "modules/rtp_rtcp/include/rtp_cvo.h"
20 #include "modules/rtp_rtcp/source/byte_io.h"
21 #include "modules/rtp_rtcp/source/playout_delay_oracle.h"
22 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
23 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
24 #include "modules/rtp_rtcp/source/rtp_sender_audio.h"
25 #include "modules/rtp_rtcp/source/rtp_sender_video.h"
26 #include "modules/rtp_rtcp/source/time_util.h"
27 #include "rtc_base/arraysize.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/logging.h"
30 #include "rtc_base/numerics/safe_minmax.h"
31 #include "rtc_base/ptr_util.h"
32 #include "rtc_base/rate_limiter.h"
33 #include "rtc_base/timeutils.h"
34 #include "rtc_base/trace_event.h"
35 #include "system_wrappers/include/field_trial.h"
36 
37 namespace webrtc {
38 
39 namespace {
40 // Max in the RFC 3550 is 255 bytes, we limit it to be modulus 32 for SRTP.
41 constexpr size_t kMaxPaddingLength = 224;
42 constexpr size_t kMinAudioPaddingLength = 50;
43 constexpr int kSendSideDelayWindowMs = 1000;
44 constexpr size_t kRtpHeaderLength = 12;
45 constexpr uint16_t kMaxInitRtpSeqNumber = 32767;  // 2^15 -1.
46 constexpr uint32_t kTimestampTicksPerMs = 90;
47 constexpr int kBitrateStatisticsWindowMs = 1000;
48 
49 constexpr size_t kMinFlexfecPacketsToStoreForPacing = 50;
50 
51 template <typename Extension>
CreateExtensionSize()52 constexpr RtpExtensionSize CreateExtensionSize() {
53   return {Extension::kId, Extension::kValueSizeBytes};
54 }
55 
56 // Size info for header extensions that might be used in padding or FEC packets.
57 constexpr RtpExtensionSize kExtensionSizes[] = {
58     CreateExtensionSize<AbsoluteSendTime>(),
59     CreateExtensionSize<TransmissionOffset>(),
60     CreateExtensionSize<TransportSequenceNumber>(),
61     CreateExtensionSize<PlayoutDelayLimits>(),
62 };
63 
FrameTypeToString(FrameType frame_type)64 const char* FrameTypeToString(FrameType frame_type) {
65   switch (frame_type) {
66     case kEmptyFrame:
67       return "empty";
68     case kAudioFrameSpeech: return "audio_speech";
69     case kAudioFrameCN: return "audio_cn";
70     case kVideoFrameKey: return "video_key";
71     case kVideoFrameDelta: return "video_delta";
72   }
73   return "";
74 }
75 
CountPacket(RtpPacketCounter * counter,const RtpPacketToSend & packet)76 void CountPacket(RtpPacketCounter* counter, const RtpPacketToSend& packet) {
77   ++counter->packets;
78   counter->header_bytes += packet.headers_size();
79   counter->padding_bytes += packet.padding_size();
80   counter->payload_bytes += packet.payload_size();
81 }
82 
83 }  // namespace
84 
RTPSender(bool audio,Clock * clock,Transport * transport,RtpPacketSender * paced_sender,FlexfecSender * flexfec_sender,TransportSequenceNumberAllocator * sequence_number_allocator,TransportFeedbackObserver * transport_feedback_observer,BitrateStatisticsObserver * bitrate_callback,FrameCountObserver * frame_count_observer,SendSideDelayObserver * send_side_delay_observer,RtcEventLog * event_log,SendPacketObserver * send_packet_observer,RateLimiter * retransmission_rate_limiter,OverheadObserver * overhead_observer)85 RTPSender::RTPSender(
86     bool audio,
87     Clock* clock,
88     Transport* transport,
89     RtpPacketSender* paced_sender,
90     FlexfecSender* flexfec_sender,
91     TransportSequenceNumberAllocator* sequence_number_allocator,
92     TransportFeedbackObserver* transport_feedback_observer,
93     BitrateStatisticsObserver* bitrate_callback,
94     FrameCountObserver* frame_count_observer,
95     SendSideDelayObserver* send_side_delay_observer,
96     RtcEventLog* event_log,
97     SendPacketObserver* send_packet_observer,
98     RateLimiter* retransmission_rate_limiter,
99     OverheadObserver* overhead_observer)
100     : clock_(clock),
101       // TODO(holmer): Remove this conversion?
102       clock_delta_ms_(clock_->TimeInMilliseconds() - rtc::TimeMillis()),
103       random_(clock_->TimeInMicroseconds()),
104       audio_configured_(audio),
105       audio_(audio ? new RTPSenderAudio(clock, this) : nullptr),
106       video_(audio ? nullptr : new RTPSenderVideo(clock, this, flexfec_sender)),
107       paced_sender_(paced_sender),
108       transport_sequence_number_allocator_(sequence_number_allocator),
109       transport_feedback_observer_(transport_feedback_observer),
110       last_capture_time_ms_sent_(0),
111       transport_(transport),
112       sending_media_(true),                   // Default to sending media.
113       max_packet_size_(IP_PACKET_SIZE - 28),  // Default is IP-v4/UDP.
114       payload_type_(-1),
115       payload_type_map_(),
116       rtp_header_extension_map_(),
117       packet_history_(clock),
118       flexfec_packet_history_(clock),
119       // Statistics
120       rtp_stats_callback_(nullptr),
121       total_bitrate_sent_(kBitrateStatisticsWindowMs,
122                           RateStatistics::kBpsScale),
123       nack_bitrate_sent_(kBitrateStatisticsWindowMs, RateStatistics::kBpsScale),
124       frame_count_observer_(frame_count_observer),
125       send_side_delay_observer_(send_side_delay_observer),
126       event_log_(event_log),
127       send_packet_observer_(send_packet_observer),
128       bitrate_callback_(bitrate_callback),
129       // RTP variables
130       remote_ssrc_(0),
131       sequence_number_forced_(false),
132       last_rtp_timestamp_(0),
133       capture_time_ms_(0),
134       last_timestamp_time_ms_(0),
135       media_has_been_sent_(false),
136       last_packet_marker_bit_(false),
137       csrcs_(),
138       rtx_(kRtxOff),
139       rtp_overhead_bytes_per_packet_(0),
140       retransmission_rate_limiter_(retransmission_rate_limiter),
141       overhead_observer_(overhead_observer),
142       send_side_bwe_with_overhead_(
143           webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")) {
144   // This random initialization is not intended to be cryptographic strong.
145   timestamp_offset_ = random_.Rand<uint32_t>();
146   // Random start, 16 bits. Can't be 0.
147   sequence_number_rtx_ = random_.Rand(1, kMaxInitRtpSeqNumber);
148   sequence_number_ = random_.Rand(1, kMaxInitRtpSeqNumber);
149 
150   // Store FlexFEC packets in the packet history data structure, so they can
151   // be found when paced.
152   if (flexfec_sender) {
153     flexfec_packet_history_.SetStorePacketsStatus(
154         true, kMinFlexfecPacketsToStoreForPacing);
155   }
156 }
157 
~RTPSender()158 RTPSender::~RTPSender() {
159   // TODO(tommi): Use a thread checker to ensure the object is created and
160   // deleted on the same thread.  At the moment this isn't possible due to
161   // voe::ChannelOwner in voice engine.  To reproduce, run:
162   // voe_auto_test --automated --gtest_filter=*MixManyChannelsForStressOpus
163 
164   // TODO(tommi,holmer): We don't grab locks in the dtor before accessing member
165   // variables but we grab them in all other methods. (what's the design?)
166   // Start documenting what thread we're on in what method so that it's easier
167   // to understand performance attributes and possibly remove locks.
168   while (!payload_type_map_.empty()) {
169     std::map<int8_t, RtpUtility::Payload*>::iterator it =
170         payload_type_map_.begin();
171     delete it->second;
172     payload_type_map_.erase(it);
173   }
174 }
175 
FecExtensionSizes()176 rtc::ArrayView<const RtpExtensionSize> RTPSender::FecExtensionSizes() {
177   return rtc::MakeArrayView(kExtensionSizes, arraysize(kExtensionSizes));
178 }
179 
ActualSendBitrateKbit() const180 uint16_t RTPSender::ActualSendBitrateKbit() const {
181   rtc::CritScope cs(&statistics_crit_);
182   return static_cast<uint16_t>(
183       total_bitrate_sent_.Rate(clock_->TimeInMilliseconds()).value_or(0) /
184       1000);
185 }
186 
VideoBitrateSent() const187 uint32_t RTPSender::VideoBitrateSent() const {
188   if (video_) {
189     return video_->VideoBitrateSent();
190   }
191   return 0;
192 }
193 
FecOverheadRate() const194 uint32_t RTPSender::FecOverheadRate() const {
195   if (video_) {
196     return video_->FecOverheadRate();
197   }
198   return 0;
199 }
200 
NackOverheadRate() const201 uint32_t RTPSender::NackOverheadRate() const {
202   rtc::CritScope cs(&statistics_crit_);
203   return nack_bitrate_sent_.Rate(clock_->TimeInMilliseconds()).value_or(0);
204 }
205 
SetRID(const char * rid)206 int32_t RTPSender::SetRID(const char* rid) {
207   rtc::CritScope lock(&send_critsect_);
208   const size_t len = (rid && rid[0]) ? strlen(rid) : 0;
209   if (len) {
210     rtpStreamId.Set(rid, len);
211   }
212   return 0;
213 }
214 
SetMId(const char * mid)215 int32_t RTPSender::SetMId(const char* mid) {
216   rtc::CritScope lock(&send_critsect_);
217   const size_t len = (mid && mid[0]) ? strlen(mid) : 0;
218   if (len) {
219     mId.Set(mid, len);
220   }
221   return 0;
222 }
223 
RegisterRtpHeaderExtension(RTPExtensionType type,uint8_t id)224 int32_t RTPSender::RegisterRtpHeaderExtension(RTPExtensionType type,
225                                               uint8_t id) {
226   rtc::CritScope lock(&send_critsect_);
227   return rtp_header_extension_map_.RegisterByType(id, type) ? 0 : -1;
228 }
229 
IsRtpHeaderExtensionRegistered(RTPExtensionType type) const230 bool RTPSender::IsRtpHeaderExtensionRegistered(RTPExtensionType type) const {
231   rtc::CritScope lock(&send_critsect_);
232   return rtp_header_extension_map_.IsRegistered(type);
233 }
234 
DeregisterRtpHeaderExtension(RTPExtensionType type)235 int32_t RTPSender::DeregisterRtpHeaderExtension(RTPExtensionType type) {
236   rtc::CritScope lock(&send_critsect_);
237   return rtp_header_extension_map_.Deregister(type);
238 }
239 
RegisterPayload(const char payload_name[RTP_PAYLOAD_NAME_SIZE],int8_t payload_number,uint32_t frequency,size_t channels,uint32_t rate)240 int32_t RTPSender::RegisterPayload(
241     const char payload_name[RTP_PAYLOAD_NAME_SIZE],
242     int8_t payload_number,
243     uint32_t frequency,
244     size_t channels,
245     uint32_t rate) {
246   RTC_DCHECK_LT(strlen(payload_name), RTP_PAYLOAD_NAME_SIZE);
247   rtc::CritScope lock(&send_critsect_);
248 
249   std::map<int8_t, RtpUtility::Payload*>::iterator it =
250       payload_type_map_.find(payload_number);
251 
252   if (payload_type_map_.end() != it) {
253     // We already use this payload type.
254     RtpUtility::Payload* payload = it->second;
255     RTC_DCHECK(payload);
256 
257     // Check if it's the same as we already have.
258     if (RtpUtility::StringCompare(
259             payload->name, payload_name, RTP_PAYLOAD_NAME_SIZE - 1)) {
260       if (audio_configured_ && payload->typeSpecific.is_audio()) {
261         auto& p = payload->typeSpecific.audio_payload();
262         if (rtc::SafeEq(p.format.clockrate_hz, frequency) &&
263             (p.rate == rate || p.rate == 0 || rate == 0)) {
264           p.rate = rate;
265           // Ensure that we update the rate if new or old is zero.
266           return 0;
267         }
268       }
269       if (!audio_configured_ && !payload->typeSpecific.is_audio()) {
270         return 0;
271       }
272     }
273     return -1;
274   }
275   int32_t ret_val = 0;
276   RtpUtility::Payload* payload = nullptr;
277   if (audio_configured_) {
278     // TODO(mflodman): Change to CreateAudioPayload and make static.
279     ret_val = audio_->RegisterAudioPayload(payload_name, payload_number,
280                                            frequency, channels, rate, &payload);
281   } else {
282     payload = video_->CreateVideoPayload(payload_name, payload_number);
283   }
284   if (payload) {
285     payload_type_map_[payload_number] = payload;
286   }
287   return ret_val;
288 }
289 
DeRegisterSendPayload(int8_t payload_type)290 int32_t RTPSender::DeRegisterSendPayload(int8_t payload_type) {
291   rtc::CritScope lock(&send_critsect_);
292 
293   std::map<int8_t, RtpUtility::Payload*>::iterator it =
294       payload_type_map_.find(payload_type);
295 
296   if (payload_type_map_.end() == it) {
297     return -1;
298   }
299   RtpUtility::Payload* payload = it->second;
300   delete payload;
301   payload_type_map_.erase(it);
302   return 0;
303 }
304 
305 // TODO(nisse): Delete this method, only used internally and by test code.
SetSendPayloadType(int8_t payload_type)306 void RTPSender::SetSendPayloadType(int8_t payload_type) {
307   rtc::CritScope lock(&send_critsect_);
308   payload_type_ = payload_type;
309 }
310 
SetMaxRtpPacketSize(size_t max_packet_size)311 void RTPSender::SetMaxRtpPacketSize(size_t max_packet_size) {
312   RTC_DCHECK_GE(max_packet_size, 100);
313   RTC_DCHECK_LE(max_packet_size, IP_PACKET_SIZE);
314   rtc::CritScope lock(&send_critsect_);
315   max_packet_size_ = max_packet_size;
316 }
317 
MaxRtpPacketSize() const318 size_t RTPSender::MaxRtpPacketSize() const {
319   return max_packet_size_;
320 }
321 
SetRtxStatus(int mode)322 void RTPSender::SetRtxStatus(int mode) {
323   rtc::CritScope lock(&send_critsect_);
324   rtx_ = mode;
325 }
326 
RtxStatus() const327 int RTPSender::RtxStatus() const {
328   rtc::CritScope lock(&send_critsect_);
329   return rtx_;
330 }
331 
SetRtxSsrc(uint32_t ssrc)332 void RTPSender::SetRtxSsrc(uint32_t ssrc) {
333   rtc::CritScope lock(&send_critsect_);
334   ssrc_rtx_.emplace(ssrc);
335 }
336 
RtxSsrc() const337 uint32_t RTPSender::RtxSsrc() const {
338   rtc::CritScope lock(&send_critsect_);
339   RTC_DCHECK(ssrc_rtx_);
340   return *ssrc_rtx_;
341 }
342 
SetRtxPayloadType(int payload_type,int associated_payload_type)343 void RTPSender::SetRtxPayloadType(int payload_type,
344                                   int associated_payload_type) {
345   rtc::CritScope lock(&send_critsect_);
346   RTC_DCHECK_LE(payload_type, 127);
347   RTC_DCHECK_LE(associated_payload_type, 127);
348   if (payload_type < 0) {
349     RTC_LOG(LS_ERROR) << "Invalid RTX payload type: " << payload_type << ".";
350     return;
351   }
352 
353   rtx_payload_type_map_[associated_payload_type] = payload_type;
354 }
355 
CheckPayloadType(int8_t payload_type,RtpVideoCodecTypes * video_type)356 int32_t RTPSender::CheckPayloadType(int8_t payload_type,
357                                     RtpVideoCodecTypes* video_type) {
358   rtc::CritScope lock(&send_critsect_);
359 
360   if (payload_type < 0) {
361     RTC_LOG(LS_ERROR) << "Invalid payload_type " << payload_type << ".";
362     return -1;
363   }
364   if (payload_type_ == payload_type) {
365     if (!audio_configured_) {
366       *video_type = video_->VideoCodecType();
367     }
368     return 0;
369   }
370   std::map<int8_t, RtpUtility::Payload*>::iterator it =
371       payload_type_map_.find(payload_type);
372   if (it == payload_type_map_.end()) {
373     RTC_LOG(LS_WARNING) << "Payload type " << static_cast<int>(payload_type)
374                         << " not registered.";
375     return -1;
376   }
377   SetSendPayloadType(payload_type);
378   RtpUtility::Payload* payload = it->second;
379   RTC_DCHECK(payload);
380   if (payload->typeSpecific.is_video() && !audio_configured_) {
381     video_->SetVideoCodecType(
382         payload->typeSpecific.video_payload().videoCodecType);
383     *video_type = payload->typeSpecific.video_payload().videoCodecType;
384   }
385   return 0;
386 }
387 
SendOutgoingData(FrameType frame_type,int8_t payload_type,uint32_t capture_timestamp,int64_t capture_time_ms,const uint8_t * payload_data,size_t payload_size,const RTPFragmentationHeader * fragmentation,const RTPVideoHeader * rtp_header,uint32_t * transport_frame_id_out,int64_t expected_retransmission_time_ms)388 bool RTPSender::SendOutgoingData(FrameType frame_type,
389                                  int8_t payload_type,
390                                  uint32_t capture_timestamp,
391                                  int64_t capture_time_ms,
392                                  const uint8_t* payload_data,
393                                  size_t payload_size,
394                                  const RTPFragmentationHeader* fragmentation,
395                                  const RTPVideoHeader* rtp_header,
396                                  uint32_t* transport_frame_id_out,
397                                  int64_t expected_retransmission_time_ms) {
398   uint32_t ssrc;
399   uint16_t sequence_number;
400   uint32_t rtp_timestamp;
401   {
402     // Drop this packet if we're not sending media packets.
403     rtc::CritScope lock(&send_critsect_);
404     RTC_DCHECK(ssrc_);
405 
406     ssrc = *ssrc_;
407     sequence_number = sequence_number_;
408     rtp_timestamp = timestamp_offset_ + capture_timestamp;
409     if (transport_frame_id_out)
410       *transport_frame_id_out = rtp_timestamp;
411     if (!sending_media_)
412       return true;
413   }
414   RtpVideoCodecTypes video_type = kRtpVideoGeneric;
415   if (CheckPayloadType(payload_type, &video_type) != 0) {
416     RTC_LOG(LS_ERROR) << "Don't send data with unknown payload type: "
417                       << static_cast<int>(payload_type) << ".";
418     return false;
419   }
420 
421   switch (frame_type) {
422     case kAudioFrameSpeech:
423     case kAudioFrameCN:
424       RTC_CHECK(audio_configured_);
425       break;
426     case kVideoFrameKey:
427     case kVideoFrameDelta:
428       RTC_CHECK(!audio_configured_);
429       break;
430     case kEmptyFrame:
431       break;
432   }
433 
434   bool result;
435   if (audio_configured_) {
436     TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type",
437                             FrameTypeToString(frame_type));
438     // The only known way to produce of RTPFragmentationHeader for audio is
439     // to use the AudioCodingModule directly.
440     RTC_DCHECK(fragmentation == nullptr);
441     result = audio_->SendAudio(frame_type, payload_type, rtp_timestamp,
442                                payload_data, payload_size, &mId);
443   } else {
444     TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms,
445                             "Send", "type", FrameTypeToString(frame_type));
446     if (frame_type == kEmptyFrame)
447       return true;
448 
449     if (rtp_header) {
450       playout_delay_oracle_.UpdateRequest(ssrc, rtp_header->playout_delay,
451                                           sequence_number);
452     }
453 
454     result = video_->SendVideo(video_type, frame_type, payload_type,
455                                rtp_timestamp, capture_time_ms, payload_data,
456                                payload_size, fragmentation, rtp_header,
457                                expected_retransmission_time_ms, &rtpStreamId,
458                                &mId);
459   }
460 
461   rtc::CritScope cs(&statistics_crit_);
462   // Note: This is currently only counting for video.
463   if (frame_type == kVideoFrameKey) {
464     ++frame_counts_.key_frames;
465   } else if (frame_type == kVideoFrameDelta) {
466     ++frame_counts_.delta_frames;
467   }
468   if (frame_count_observer_) {
469     frame_count_observer_->FrameCountUpdated(frame_counts_, ssrc);
470   }
471 
472   return result;
473 }
474 
TrySendRedundantPayloads(size_t bytes_to_send,const PacedPacketInfo & pacing_info)475 size_t RTPSender::TrySendRedundantPayloads(size_t bytes_to_send,
476                                            const PacedPacketInfo& pacing_info) {
477   {
478     rtc::CritScope lock(&send_critsect_);
479     if (!sending_media_)
480       return 0;
481     if ((rtx_ & kRtxRedundantPayloads) == 0)
482       return 0;
483   }
484 
485   int bytes_left = static_cast<int>(bytes_to_send);
486   while (bytes_left > 0) {
487     std::unique_ptr<RtpPacketToSend> packet =
488         packet_history_.GetBestFittingPacket(bytes_left);
489     if (!packet)
490       break;
491     size_t payload_size = packet->payload_size();
492     if (!PrepareAndSendPacket(std::move(packet), true, false, pacing_info))
493       break;
494     bytes_left -= payload_size;
495   }
496   return bytes_to_send - bytes_left;
497 }
498 
SendPadData(size_t bytes,const PacedPacketInfo & pacing_info)499 size_t RTPSender::SendPadData(size_t bytes,
500                               const PacedPacketInfo& pacing_info) {
501   size_t padding_bytes_in_packet;
502   size_t max_payload_size = max_packet_size_ - RtpHeaderLength();
503 
504   if (audio_configured_) {
505     // Allow smaller padding packets for audio.
506     padding_bytes_in_packet = rtc::SafeClamp<size_t>(
507         bytes, kMinAudioPaddingLength,
508         rtc::SafeMin(max_payload_size, kMaxPaddingLength));
509   } else {
510     // Always send full padding packets. This is accounted for by the
511     // RtpPacketSender, which will make sure we don't send too much padding even
512     // if a single packet is larger than requested.
513     // We do this to avoid frequently sending small packets on higher bitrates.
514     padding_bytes_in_packet =
515         rtc::SafeMin<size_t>(max_payload_size, kMaxPaddingLength);
516   }
517   size_t bytes_sent = 0;
518   while (bytes_sent < bytes) {
519     int64_t now_ms = clock_->TimeInMilliseconds();
520     uint32_t ssrc;
521     uint32_t timestamp;
522     int64_t capture_time_ms;
523     uint16_t sequence_number;
524     int payload_type;
525     bool over_rtx;
526     {
527       rtc::CritScope lock(&send_critsect_);
528       if (!sending_media_)
529         break;
530       timestamp = last_rtp_timestamp_;
531       capture_time_ms = capture_time_ms_;
532       if (rtx_ == kRtxOff) {
533         if (payload_type_ == -1)
534           break;
535         // Without RTX we can't send padding in the middle of frames.
536         // For audio marker bits doesn't mark the end of a frame and frames
537         // are usually a single packet, so for now we don't apply this rule
538         // for audio.
539         if (!audio_configured_ && !last_packet_marker_bit_) {
540           break;
541         }
542         if (!ssrc_) {
543           RTC_LOG(LS_ERROR) << "SSRC unset.";
544           return 0;
545         }
546 
547         RTC_DCHECK(ssrc_);
548         ssrc = *ssrc_;
549 
550         sequence_number = sequence_number_;
551         ++sequence_number_;
552         payload_type = payload_type_;
553         over_rtx = false;
554       } else {
555         // Without abs-send-time or transport sequence number a media packet
556         // must be sent before padding so that the timestamps used for
557         // estimation are correct.
558         if (!media_has_been_sent_ &&
559             !(rtp_header_extension_map_.IsRegistered(AbsoluteSendTime::kId) ||
560               (rtp_header_extension_map_.IsRegistered(
561                    TransportSequenceNumber::kId) &&
562                transport_sequence_number_allocator_))) {
563           break;
564         }
565         // Only change change the timestamp of padding packets sent over RTX.
566         // Padding only packets over RTP has to be sent as part of a media
567         // frame (and therefore the same timestamp).
568         if (last_timestamp_time_ms_ > 0) {
569           timestamp +=
570               (now_ms - last_timestamp_time_ms_) * kTimestampTicksPerMs;
571           if (capture_time_ms > 0) {
572             capture_time_ms += (now_ms - last_timestamp_time_ms_);
573           }
574         }
575         if (!ssrc_rtx_) {
576           RTC_LOG(LS_ERROR) << "RTX SSRC unset.";
577           return 0;
578         }
579         RTC_DCHECK(ssrc_rtx_);
580         ssrc = *ssrc_rtx_;
581         sequence_number = sequence_number_rtx_;
582         ++sequence_number_rtx_;
583         payload_type = rtx_payload_type_map_.begin()->second;
584         over_rtx = true;
585       }
586     }
587 
588     std::unique_ptr<RtpPacketToSend> padding_packet(new RtpPacketToSend(&rtp_header_extension_map_));
589     padding_packet->SetPayloadType(payload_type);
590     padding_packet->SetMarker(false);
591     padding_packet->SetSequenceNumber(sequence_number);
592     padding_packet->SetTimestamp(timestamp);
593     padding_packet->SetSsrc(ssrc);
594 
595     if (capture_time_ms > 0) {
596       padding_packet->SetExtension<TransmissionOffset>(
597           (now_ms - capture_time_ms) * kTimestampTicksPerMs);
598     }
599     padding_packet->SetExtension<AbsoluteSendTime>(
600         AbsoluteSendTime::MsTo24Bits(now_ms));
601     PacketOptions options;
602     bool has_transport_seq_num =
603         UpdateTransportSequenceNumber(padding_packet.get(), &options.packet_id);
604     padding_packet->SetPadding(padding_bytes_in_packet, &random_);
605 
606     if (has_transport_seq_num) {
607       AddPacketToTransportFeedback(options.packet_id, *padding_packet,
608                                    pacing_info);
609     }
610 
611     if (!SendPacketToNetwork(*padding_packet, options, pacing_info))
612       break;
613     bytes_sent += padding_bytes_in_packet;
614     UpdateRtpStats(*padding_packet, over_rtx, false);
615 
616     packet_history_.PutRtpPacket(std::move(padding_packet),
617                                  kAllowRetransmission,
618                                  true);
619   }
620 
621   return bytes_sent;
622 }
623 
SetStorePacketsStatus(bool enable,uint16_t number_to_store)624 void RTPSender::SetStorePacketsStatus(bool enable, uint16_t number_to_store) {
625   packet_history_.SetStorePacketsStatus(enable, number_to_store);
626 }
627 
StorePackets() const628 bool RTPSender::StorePackets() const {
629   return packet_history_.StorePackets();
630 }
631 
ReSendPacket(uint16_t packet_id,int64_t min_resend_time)632 int32_t RTPSender::ReSendPacket(uint16_t packet_id, int64_t min_resend_time) {
633   std::unique_ptr<RtpPacketToSend> packet =
634       packet_history_.GetPacketAndSetSendTime(packet_id, min_resend_time, true);
635   if (!packet) {
636     // Packet not found.
637     return 0;
638   }
639 
640   // Check if we're overusing retransmission bitrate.
641   // TODO(sprang): Add histograms for nack success or failure reasons.
642   RTC_DCHECK(retransmission_rate_limiter_);
643   if (!retransmission_rate_limiter_->TryUseRate(packet->size()))
644     return -1;
645 
646   if (paced_sender_) {
647     // Convert from TickTime to Clock since capture_time_ms is based on
648     // TickTime.
649     int64_t corrected_capture_tims_ms =
650         packet->capture_time_ms() + clock_delta_ms_;
651     paced_sender_->InsertPacket(RtpPacketSender::kNormalPriority,
652                                 packet->Ssrc(), packet->SequenceNumber(),
653                                 corrected_capture_tims_ms,
654                                 packet->payload_size(), true);
655 
656     return packet->size();
657   }
658   bool rtx = (RtxStatus() & kRtxRetransmitted) > 0;
659   int32_t packet_size = static_cast<int32_t>(packet->size());
660   if (!PrepareAndSendPacket(std::move(packet), rtx, true, PacedPacketInfo()))
661     return -1;
662   return packet_size;
663 }
664 
SendPacketToNetwork(const RtpPacketToSend & packet,const PacketOptions & options,const PacedPacketInfo & pacing_info)665 bool RTPSender::SendPacketToNetwork(const RtpPacketToSend& packet,
666                                     const PacketOptions& options,
667                                     const PacedPacketInfo& pacing_info) {
668   int bytes_sent = -1;
669   if (transport_) {
670     UpdateRtpOverhead(packet);
671     bytes_sent = transport_->SendRtp(packet.data(), packet.size(), options)
672                      ? static_cast<int>(packet.size())
673                      : -1;
674     if (event_log_ && bytes_sent > 0) {
675       event_log_->Log(rtc::MakeUnique<RtcEventRtpPacketOutgoing>(
676           packet, pacing_info.probe_cluster_id));
677     }
678   }
679   TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
680                        "RTPSender::SendPacketToNetwork", "size", packet.size(),
681                        "sent", bytes_sent);
682   // TODO(pwestin): Add a separate bitrate for sent bitrate after pacer.
683   if (bytes_sent <= 0) {
684     RTC_LOG(LS_WARNING) << "Transport failed to send packet.";
685     return false;
686   }
687   return true;
688 }
689 
SelectiveRetransmissions() const690 int RTPSender::SelectiveRetransmissions() const {
691   if (!video_)
692     return -1;
693   return video_->SelectiveRetransmissions();
694 }
695 
SetSelectiveRetransmissions(uint8_t settings)696 int RTPSender::SetSelectiveRetransmissions(uint8_t settings) {
697   if (!video_)
698     return -1;
699   video_->SetSelectiveRetransmissions(settings);
700   return 0;
701 }
702 
OnReceivedNack(const std::vector<uint16_t> & nack_sequence_numbers,int64_t avg_rtt)703 void RTPSender::OnReceivedNack(
704     const std::vector<uint16_t>& nack_sequence_numbers,
705     int64_t avg_rtt) {
706   TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
707                "RTPSender::OnReceivedNACK", "num_seqnum",
708                nack_sequence_numbers.size(), "avg_rtt", avg_rtt);
709   for (uint16_t seq_no : nack_sequence_numbers) {
710     const int32_t bytes_sent = ReSendPacket(seq_no, 5 + avg_rtt);
711     if (bytes_sent < 0) {
712       // Failed to send one Sequence number. Give up the rest in this nack.
713       RTC_LOG(LS_WARNING) << "Failed resending RTP packet " << seq_no
714                           << ", Discard rest of packets.";
715       break;
716     }
717   }
718 }
719 
OnReceivedRtcpReportBlocks(const ReportBlockList & report_blocks)720 void RTPSender::OnReceivedRtcpReportBlocks(
721     const ReportBlockList& report_blocks) {
722   playout_delay_oracle_.OnReceivedRtcpReportBlocks(report_blocks);
723 }
724 
725 // Called from pacer when we can send the packet.
TimeToSendPacket(uint32_t ssrc,uint16_t sequence_number,int64_t capture_time_ms,bool retransmission,const PacedPacketInfo & pacing_info)726 bool RTPSender::TimeToSendPacket(uint32_t ssrc,
727                                  uint16_t sequence_number,
728                                  int64_t capture_time_ms,
729                                  bool retransmission,
730                                  const PacedPacketInfo& pacing_info) {
731   if (!SendingMedia())
732     return true;
733 
734   std::unique_ptr<RtpPacketToSend> packet;
735   if (ssrc == SSRC()) {
736     packet = packet_history_.GetPacketAndSetSendTime(sequence_number, 0,
737                                                      retransmission);
738   } else if (ssrc == FlexfecSsrc()) {
739     packet = flexfec_packet_history_.GetPacketAndSetSendTime(sequence_number, 0,
740                                                              retransmission);
741   }
742 
743   if (!packet) {
744     // Packet cannot be found.
745     return true;
746   }
747 
748   return PrepareAndSendPacket(
749       std::move(packet),
750       retransmission && (RtxStatus() & kRtxRetransmitted) > 0, retransmission,
751       pacing_info);
752 }
753 
PrepareAndSendPacket(std::unique_ptr<RtpPacketToSend> packet,bool send_over_rtx,bool is_retransmit,const PacedPacketInfo & pacing_info)754 bool RTPSender::PrepareAndSendPacket(std::unique_ptr<RtpPacketToSend> packet,
755                                      bool send_over_rtx,
756                                      bool is_retransmit,
757                                      const PacedPacketInfo& pacing_info) {
758   RTC_DCHECK(packet);
759   int64_t capture_time_ms = packet->capture_time_ms();
760   RtpPacketToSend* packet_to_send = packet.get();
761 
762   if (!is_retransmit && packet->Marker()) {
763     TRACE_EVENT_ASYNC_END0(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), "PacedSend",
764                            capture_time_ms);
765   }
766 
767   TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
768                        "PrepareAndSendPacket", "timestamp", packet->Timestamp(),
769                        "seqnum", packet->SequenceNumber());
770 
771   std::unique_ptr<RtpPacketToSend> packet_rtx;
772   if (send_over_rtx) {
773     packet_rtx = BuildRtxPacket(*packet);
774     if (!packet_rtx)
775       return false;
776     packet_to_send = packet_rtx.get();
777   }
778 
779   // Bug webrtc:7859. While FEC is invoked from rtp_sender_video, and not after
780   // the pacer, these modifications of the header below are happening after the
781   // FEC protection packets are calculated. This will corrupt recovered packets
782   // at the same place. It's not an issue for extensions, which are present in
783   // all the packets (their content just may be incorrect on recovered packets).
784   // In case of VideoTimingExtension, since it's present not in every packet,
785   // data after rtp header may be corrupted if these packets are protected by
786   // the FEC.
787   int64_t now_ms = clock_->TimeInMilliseconds();
788   int64_t diff_ms = now_ms - capture_time_ms;
789   packet_to_send->SetExtension<TransmissionOffset>(kTimestampTicksPerMs *
790                                                    diff_ms);
791   packet_to_send->SetExtension<AbsoluteSendTime>(
792       AbsoluteSendTime::MsTo24Bits(now_ms));
793 
794   if (packet_to_send->HasExtension<VideoTimingExtension>())
795     packet_to_send->set_pacer_exit_time_ms(now_ms);
796 
797   PacketOptions options;
798   if (UpdateTransportSequenceNumber(packet_to_send, &options.packet_id)) {
799     AddPacketToTransportFeedback(options.packet_id, *packet_to_send,
800                                  pacing_info);
801   }
802 
803   if (!is_retransmit && !send_over_rtx) {
804     UpdateDelayStatistics(packet->capture_time_ms(), now_ms);
805     UpdateOnSendPacket(options.packet_id, packet->capture_time_ms(),
806                        packet->Ssrc());
807   }
808 
809   if (!SendPacketToNetwork(*packet_to_send, options, pacing_info))
810     return false;
811 
812   {
813     rtc::CritScope lock(&send_critsect_);
814     media_has_been_sent_ = true;
815   }
816   UpdateRtpStats(*packet_to_send, send_over_rtx, is_retransmit);
817   return true;
818 }
819 
UpdateRtpStats(const RtpPacketToSend & packet,bool is_rtx,bool is_retransmit)820 void RTPSender::UpdateRtpStats(const RtpPacketToSend& packet,
821                                bool is_rtx,
822                                bool is_retransmit) {
823   int64_t now_ms = clock_->TimeInMilliseconds();
824 
825   rtc::CritScope lock(&statistics_crit_);
826   StreamDataCounters* counters = is_rtx ? &rtx_rtp_stats_ : &rtp_stats_;
827 
828   total_bitrate_sent_.Update(packet.size(), now_ms);
829 
830   if (counters->first_packet_time_ms == -1)
831     counters->first_packet_time_ms = now_ms;
832 
833   if (IsFecPacket(packet))
834     CountPacket(&counters->fec, packet);
835 
836   if (is_retransmit) {
837     CountPacket(&counters->retransmitted, packet);
838     nack_bitrate_sent_.Update(packet.size(), now_ms);
839   }
840   CountPacket(&counters->transmitted, packet);
841 
842   if (rtp_stats_callback_)
843     rtp_stats_callback_->DataCountersUpdated(*counters, packet.Ssrc());
844 }
845 
IsFecPacket(const RtpPacketToSend & packet) const846 bool RTPSender::IsFecPacket(const RtpPacketToSend& packet) const {
847   if (!video_)
848     return false;
849 
850   // FlexFEC.
851   if (packet.Ssrc() == FlexfecSsrc())
852     return true;
853 
854   // RED+ULPFEC.
855   int pt_red;
856   int pt_fec;
857   video_->GetUlpfecConfig(&pt_red, &pt_fec);
858   return static_cast<int>(packet.PayloadType()) == pt_red &&
859          static_cast<int>(packet.payload()[0]) == pt_fec;
860 }
861 
TimeToSendPadding(size_t bytes,const PacedPacketInfo & pacing_info)862 size_t RTPSender::TimeToSendPadding(size_t bytes,
863                                     const PacedPacketInfo& pacing_info) {
864   if (bytes == 0)
865     return 0;
866   size_t bytes_sent = TrySendRedundantPayloads(bytes, pacing_info);
867   if (bytes_sent < bytes)
868     bytes_sent += SendPadData(bytes - bytes_sent, pacing_info);
869   return bytes_sent;
870 }
871 
SendToNetwork(std::unique_ptr<RtpPacketToSend> packet,StorageType storage,RtpPacketSender::Priority priority)872 bool RTPSender::SendToNetwork(std::unique_ptr<RtpPacketToSend> packet,
873                               StorageType storage,
874                               RtpPacketSender::Priority priority) {
875   RTC_DCHECK(packet);
876   int64_t now_ms = clock_->TimeInMilliseconds();
877 
878   // |capture_time_ms| <= 0 is considered invalid.
879   // TODO(holmer): This should be changed all over Video Engine so that negative
880   // time is consider invalid, while 0 is considered a valid time.
881   if (packet->capture_time_ms() > 0) {
882     packet->SetExtension<TransmissionOffset>(
883         kTimestampTicksPerMs * (now_ms - packet->capture_time_ms()));
884     if (packet->HasExtension<VideoTimingExtension>())
885       packet->set_pacer_exit_time_ms(now_ms);
886   }
887   packet->SetExtension<AbsoluteSendTime>(AbsoluteSendTime::MsTo24Bits(now_ms));
888 
889   if (video_) {
890     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms,
891                                     ActualSendBitrateKbit(), packet->Ssrc());
892     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoFecBitrate_kbps", now_ms,
893                                     FecOverheadRate() / 1000, packet->Ssrc());
894     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoNackBitrate_kbps", now_ms,
895                                     NackOverheadRate() / 1000, packet->Ssrc());
896   } else {
897     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioTotBitrate_kbps", now_ms,
898                                     ActualSendBitrateKbit(), packet->Ssrc());
899     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioNackBitrate_kbps", now_ms,
900                                     NackOverheadRate() / 1000, packet->Ssrc());
901   }
902 
903   uint32_t ssrc = packet->Ssrc();
904   rtc::Optional<uint32_t> flexfec_ssrc = FlexfecSsrc();
905   if (paced_sender_) {
906     uint16_t seq_no = packet->SequenceNumber();
907     // Correct offset between implementations of millisecond time stamps in
908     // TickTime and Clock.
909     int64_t corrected_time_ms = packet->capture_time_ms() + clock_delta_ms_;
910     size_t payload_length = packet->payload_size();
911     if (ssrc == flexfec_ssrc) {
912       // Store FlexFEC packets in the history here, so they can be found
913       // when the pacer calls TimeToSendPacket.
914       flexfec_packet_history_.PutRtpPacket(std::move(packet), storage, false);
915     } else {
916       packet_history_.PutRtpPacket(std::move(packet), storage, false);
917     }
918 
919     paced_sender_->InsertPacket(priority, ssrc, seq_no, corrected_time_ms,
920                                 payload_length, false);
921     if (last_capture_time_ms_sent_ == 0 ||
922         corrected_time_ms > last_capture_time_ms_sent_) {
923       last_capture_time_ms_sent_ = corrected_time_ms;
924       TRACE_EVENT_ASYNC_BEGIN1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
925                                "PacedSend", corrected_time_ms,
926                                "capture_time_ms", corrected_time_ms);
927     }
928     return true;
929   }
930 
931   PacketOptions options;
932   if (UpdateTransportSequenceNumber(packet.get(), &options.packet_id)) {
933     AddPacketToTransportFeedback(options.packet_id, *packet.get(),
934                                  PacedPacketInfo());
935   }
936 
937   UpdateDelayStatistics(packet->capture_time_ms(), now_ms);
938   UpdateOnSendPacket(options.packet_id, packet->capture_time_ms(),
939                      packet->Ssrc());
940 
941   bool sent = SendPacketToNetwork(*packet, options, PacedPacketInfo());
942 
943   if (sent) {
944     {
945       rtc::CritScope lock(&send_critsect_);
946       media_has_been_sent_ = true;
947     }
948     UpdateRtpStats(*packet, false, false);
949   }
950 
951   // To support retransmissions, we store the media packet as sent in the
952   // packet history (even if send failed).
953   if (storage == kAllowRetransmission) {
954     // TODO(brandtr): Uncomment the DCHECK line below when |ssrc_| cannot
955     // change after the first packet has been sent. For more details, see
956     // https://bugs.chromium.org/p/webrtc/issues/detail?id=6887.
957     // RTC_DCHECK_EQ(ssrc, SSRC());
958     packet_history_.PutRtpPacket(std::move(packet), storage, true);
959   }
960 
961   return sent;
962 }
963 
UpdateDelayStatistics(int64_t capture_time_ms,int64_t now_ms)964 void RTPSender::UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms) {
965   if (!send_side_delay_observer_ || capture_time_ms <= 0)
966     return;
967 
968   uint32_t ssrc;
969   int64_t avg_delay_ms = 0;
970   int max_delay_ms = 0;
971   {
972     rtc::CritScope lock(&send_critsect_);
973     if (!ssrc_)
974       return;
975     ssrc = *ssrc_;
976   }
977   {
978     rtc::CritScope cs(&statistics_crit_);
979     // TODO(holmer): Compute this iteratively instead.
980     send_delays_[now_ms] = now_ms - capture_time_ms;
981     send_delays_.erase(send_delays_.begin(),
982                        send_delays_.lower_bound(now_ms -
983                        kSendSideDelayWindowMs));
984     int num_delays = 0;
985     for (auto it = send_delays_.upper_bound(now_ms - kSendSideDelayWindowMs);
986          it != send_delays_.end(); ++it) {
987       max_delay_ms = std::max(max_delay_ms, it->second);
988       avg_delay_ms += it->second;
989       ++num_delays;
990     }
991     if (num_delays == 0)
992       return;
993     avg_delay_ms = (avg_delay_ms + num_delays / 2) / num_delays;
994   }
995   send_side_delay_observer_->SendSideDelayUpdated(
996       rtc::dchecked_cast<int>(avg_delay_ms), max_delay_ms, ssrc);
997 }
998 
UpdateOnSendPacket(int packet_id,int64_t capture_time_ms,uint32_t ssrc)999 void RTPSender::UpdateOnSendPacket(int packet_id,
1000                                    int64_t capture_time_ms,
1001                                    uint32_t ssrc) {
1002   if (!send_packet_observer_ || capture_time_ms <= 0 || packet_id == -1)
1003     return;
1004 
1005   send_packet_observer_->OnSendPacket(packet_id, capture_time_ms, ssrc);
1006 }
1007 
ProcessBitrate()1008 void RTPSender::ProcessBitrate() {
1009   if (!bitrate_callback_)
1010     return;
1011   int64_t now_ms = clock_->TimeInMilliseconds();
1012   uint32_t ssrc;
1013   {
1014     rtc::CritScope lock(&send_critsect_);
1015     if (!ssrc_)
1016       return;
1017     ssrc = *ssrc_;
1018   }
1019 
1020   rtc::CritScope lock(&statistics_crit_);
1021   bitrate_callback_->Notify(total_bitrate_sent_.Rate(now_ms).value_or(0),
1022                             nack_bitrate_sent_.Rate(now_ms).value_or(0), ssrc);
1023 }
1024 
RtpHeaderLength() const1025 size_t RTPSender::RtpHeaderLength() const {
1026   rtc::CritScope lock(&send_critsect_);
1027   size_t rtp_header_length = kRtpHeaderLength;
1028   rtp_header_length += sizeof(uint32_t) * csrcs_.size();
1029   rtp_header_length +=
1030       rtp_header_extension_map_.GetTotalLengthInBytes(kExtensionSizes);
1031   return rtp_header_length;
1032 }
1033 
AllocateSequenceNumber(uint16_t packets_to_send)1034 uint16_t RTPSender::AllocateSequenceNumber(uint16_t packets_to_send) {
1035   rtc::CritScope lock(&send_critsect_);
1036   uint16_t first_allocated_sequence_number = sequence_number_;
1037   sequence_number_ += packets_to_send;
1038   return first_allocated_sequence_number;
1039 }
1040 
GetDataCounters(StreamDataCounters * rtp_stats,StreamDataCounters * rtx_stats) const1041 void RTPSender::GetDataCounters(StreamDataCounters* rtp_stats,
1042                                 StreamDataCounters* rtx_stats) const {
1043   rtc::CritScope lock(&statistics_crit_);
1044   *rtp_stats = rtp_stats_;
1045   *rtx_stats = rtx_rtp_stats_;
1046 }
1047 
AllocatePacket() const1048 std::unique_ptr<RtpPacketToSend> RTPSender::AllocatePacket() const {
1049   rtc::CritScope lock(&send_critsect_);
1050   std::unique_ptr<RtpPacketToSend> packet(
1051       new RtpPacketToSend(&rtp_header_extension_map_, max_packet_size_));
1052   RTC_DCHECK(ssrc_);
1053   packet->SetSsrc(*ssrc_);
1054   packet->SetCsrcs(csrcs_);
1055   // Reserve extensions, if registered, RtpSender set in SendToNetwork.
1056   packet->ReserveExtension<AbsoluteSendTime>();
1057   packet->ReserveExtension<TransmissionOffset>();
1058   packet->ReserveExtension<TransportSequenceNumber>();
1059   if (playout_delay_oracle_.send_playout_delay()) {
1060     packet->SetExtension<PlayoutDelayLimits>(
1061         playout_delay_oracle_.playout_delay());
1062   }
1063   return packet;
1064 }
1065 
AssignSequenceNumber(RtpPacketToSend * packet)1066 bool RTPSender::AssignSequenceNumber(RtpPacketToSend* packet) {
1067   rtc::CritScope lock(&send_critsect_);
1068   if (!sending_media_)
1069     return false;
1070   RTC_DCHECK(packet->Ssrc() == ssrc_);
1071   packet->SetSequenceNumber(sequence_number_++);
1072 
1073   // Remember marker bit to determine if padding can be inserted with
1074   // sequence number following |packet|.
1075   last_packet_marker_bit_ = packet->Marker();
1076   // Save timestamps to generate timestamp field and extensions for the padding.
1077   last_rtp_timestamp_ = packet->Timestamp();
1078   last_timestamp_time_ms_ = clock_->TimeInMilliseconds();
1079   capture_time_ms_ = packet->capture_time_ms();
1080   return true;
1081 }
1082 
UpdateTransportSequenceNumber(RtpPacketToSend * packet,int * packet_id) const1083 bool RTPSender::UpdateTransportSequenceNumber(RtpPacketToSend* packet,
1084                                               int* packet_id) const {
1085   RTC_DCHECK(packet);
1086   RTC_DCHECK(packet_id);
1087   rtc::CritScope lock(&send_critsect_);
1088   if (!rtp_header_extension_map_.IsRegistered(TransportSequenceNumber::kId))
1089     return false;
1090 
1091   if (!transport_sequence_number_allocator_)
1092     return false;
1093 
1094   *packet_id = transport_sequence_number_allocator_->AllocateSequenceNumber();
1095 
1096   if (!packet->SetExtension<TransportSequenceNumber>(*packet_id))
1097     return false;
1098 
1099   return true;
1100 }
1101 
SetSendingMediaStatus(bool enabled)1102 void RTPSender::SetSendingMediaStatus(bool enabled) {
1103   rtc::CritScope lock(&send_critsect_);
1104   sending_media_ = enabled;
1105 }
1106 
SendingMedia() const1107 bool RTPSender::SendingMedia() const {
1108   rtc::CritScope lock(&send_critsect_);
1109   return sending_media_;
1110 }
1111 
SetTimestampOffset(uint32_t timestamp)1112 void RTPSender::SetTimestampOffset(uint32_t timestamp) {
1113   rtc::CritScope lock(&send_critsect_);
1114   timestamp_offset_ = timestamp;
1115 }
1116 
TimestampOffset() const1117 uint32_t RTPSender::TimestampOffset() const {
1118   rtc::CritScope lock(&send_critsect_);
1119   return timestamp_offset_;
1120 }
1121 
SetSSRC(uint32_t ssrc)1122 void RTPSender::SetSSRC(uint32_t ssrc) {
1123   // This is configured via the API.
1124   rtc::CritScope lock(&send_critsect_);
1125 
1126   if (ssrc_ == ssrc) {
1127     return;  // Since it's same ssrc, don't reset anything.
1128   }
1129   ssrc_.emplace(ssrc);
1130   if (!sequence_number_forced_) {
1131     sequence_number_ = random_.Rand(1, kMaxInitRtpSeqNumber);
1132   }
1133 }
1134 
SSRC() const1135 uint32_t RTPSender::SSRC() const {
1136   rtc::CritScope lock(&send_critsect_);
1137   RTC_DCHECK(ssrc_);
1138   return *ssrc_;
1139 }
1140 
FlexfecSsrc() const1141 rtc::Optional<uint32_t> RTPSender::FlexfecSsrc() const {
1142   if (video_) {
1143     return video_->FlexfecSsrc();
1144   }
1145   return rtc::nullopt;
1146 }
1147 
SetCsrcs(const std::vector<uint32_t> & csrcs)1148 void RTPSender::SetCsrcs(const std::vector<uint32_t>& csrcs) {
1149   RTC_DCHECK_LE(csrcs.size(), kRtpCsrcSize);
1150   rtc::CritScope lock(&send_critsect_);
1151   csrcs_ = csrcs;
1152 }
1153 
SetSequenceNumber(uint16_t seq)1154 void RTPSender::SetSequenceNumber(uint16_t seq) {
1155   rtc::CritScope lock(&send_critsect_);
1156   sequence_number_forced_ = true;
1157   sequence_number_ = seq;
1158 }
1159 
SequenceNumber() const1160 uint16_t RTPSender::SequenceNumber() const {
1161   rtc::CritScope lock(&send_critsect_);
1162   return sequence_number_;
1163 }
1164 
1165 // Audio.
SendTelephoneEvent(uint8_t key,uint16_t time_ms,uint8_t level)1166 int32_t RTPSender::SendTelephoneEvent(uint8_t key,
1167                                       uint16_t time_ms,
1168                                       uint8_t level) {
1169   if (!audio_configured_) {
1170     return -1;
1171   }
1172   return audio_->SendTelephoneEvent(key, time_ms, level);
1173 }
1174 
SetAudioLevel(uint8_t level_d_bov)1175 int32_t RTPSender::SetAudioLevel(uint8_t level_d_bov) {
1176   return audio_->SetAudioLevel(level_d_bov);
1177 }
1178 
VideoCodecType() const1179 RtpVideoCodecTypes RTPSender::VideoCodecType() const {
1180   RTC_DCHECK(!audio_configured_) << "Sender is an audio stream!";
1181   return video_->VideoCodecType();
1182 }
1183 
SetUlpfecConfig(int red_payload_type,int ulpfec_payload_type)1184 void RTPSender::SetUlpfecConfig(int red_payload_type, int ulpfec_payload_type) {
1185   RTC_DCHECK(!audio_configured_);
1186   video_->SetUlpfecConfig(red_payload_type, ulpfec_payload_type);
1187 }
1188 
SetFecParameters(const FecProtectionParams & delta_params,const FecProtectionParams & key_params)1189 bool RTPSender::SetFecParameters(const FecProtectionParams& delta_params,
1190                                  const FecProtectionParams& key_params) {
1191   if (audio_configured_) {
1192     return false;
1193   }
1194   video_->SetFecParameters(delta_params, key_params);
1195   return true;
1196 }
1197 
CopyHeaderAndExtensionsToRtxPacket(const RtpPacketToSend & packet,RtpPacketToSend * rtx_packet)1198 static void CopyHeaderAndExtensionsToRtxPacket(const RtpPacketToSend& packet,
1199                                                RtpPacketToSend* rtx_packet) {
1200   // Set the relevant fixed packet headers. The following are not set:
1201   // * Payload type - it is replaced in rtx packets.
1202   // * Sequence number - RTX has a separate sequence numbering.
1203   // * SSRC - RTX stream has its own SSRC.
1204   rtx_packet->SetMarker(packet.Marker());
1205   rtx_packet->SetTimestamp(packet.Timestamp());
1206 
1207   // Set the variable fields in the packet header:
1208   // * CSRCs - must be set before header extensions.
1209   // * Header extensions - replace Rid header with RepairedRid header.
1210   const std::vector<uint32_t> csrcs = packet.Csrcs();
1211   rtx_packet->SetCsrcs(csrcs);
1212   for (int extension_num = kRtpExtensionNone + 1;
1213        extension_num < kRtpExtensionNumberOfExtensions; ++extension_num) {
1214     auto extension = static_cast<RTPExtensionType>(extension_num);
1215 
1216     // Stream ID header extensions (MID, RSID) are sent per-SSRC. Since RTX
1217     // operates on a different SSRC, the presence and values of these header
1218     // extensions should be determined separately and not blindly copied.
1219     if (extension == kRtpExtensionMid ||
1220         extension == kRtpExtensionRtpStreamId) {
1221       continue;
1222     }
1223 
1224     rtc::ArrayView<const uint8_t> source = packet.FindExtension(extension);
1225 
1226     // Empty extensions should be supported, so not checking |source.empty()|.
1227     // TODO: But this does not work in Mozilla's version of libwebrtc. Remove
1228     // this check with the next update from tip of libwebrtc.
1229     if (source.empty()) {
1230       continue;
1231     }
1232 
1233     rtc::ArrayView<uint8_t> destination =
1234         rtx_packet->AllocateExtension(extension, source.size());
1235 
1236     // Could happen if any:
1237     // 1. Extension has 0 length.
1238     // 2. Extension is not registered in destination.
1239     // 3. Allocating extension in destination failed.
1240     if (destination.empty() || source.size() != destination.size()) {
1241       continue;
1242     }
1243 
1244     std::memcpy(destination.begin(), source.begin(), destination.size());
1245   }
1246 }
1247 
BuildRtxPacket(const RtpPacketToSend & packet)1248 std::unique_ptr<RtpPacketToSend> RTPSender::BuildRtxPacket(
1249     const RtpPacketToSend& packet) {
1250   // TODO(danilchap): Create rtx packet with extra capacity for SRTP
1251   // when transport interface would be updated to take buffer class.
1252   std::unique_ptr<RtpPacketToSend> rtx_packet(new RtpPacketToSend(
1253       &rtp_header_extension_map_, max_packet_size_));
1254   // Add original RTP header.
1255   {
1256     rtc::CritScope lock(&send_critsect_);
1257     if (!sending_media_)
1258       return nullptr;
1259 
1260     RTC_DCHECK(ssrc_rtx_);
1261 
1262     // Replace payload type.
1263     auto kv = rtx_payload_type_map_.find(packet.PayloadType());
1264     if (kv == rtx_payload_type_map_.end())
1265       return nullptr;
1266     rtx_packet->SetPayloadType(kv->second);
1267 
1268     // Replace sequence number.
1269     rtx_packet->SetSequenceNumber(sequence_number_rtx_++);
1270 
1271     // Replace SSRC.
1272     rtx_packet->SetSsrc(*ssrc_rtx_);
1273 
1274     CopyHeaderAndExtensionsToRtxPacket(packet, rtx_packet.get());
1275 
1276     // Copy rtp-stream-id from packet to repaired-rtp-stream-id
1277     if (rtp_header_extension_map_.IsRegistered(kRtpExtensionRtpStreamId) &&
1278         rtp_header_extension_map_.IsRegistered(kRtpExtensionRepairedRtpStreamId)) {
1279       std::string rid;
1280       if (packet.GetExtension<RtpStreamId>(&rid)) {
1281         rtx_packet->SetExtension<RepairedRtpStreamId>(rid);
1282       }
1283     }
1284 
1285     // Copy mid from packet
1286     if (rtp_header_extension_map_.IsRegistered(kRtpExtensionMid)) {
1287       std::string mid;
1288       if (packet.GetExtension<RtpMid>(&mid)) {
1289         rtx_packet->SetExtension<RtpMid>(mid);
1290       }
1291     }
1292   }
1293   uint8_t* rtx_payload =
1294       rtx_packet->AllocatePayload(packet.payload_size() + kRtxHeaderSize);
1295   RTC_DCHECK(rtx_payload);
1296   // Add OSN (original sequence number).
1297   ByteWriter<uint16_t>::WriteBigEndian(rtx_payload, packet.SequenceNumber());
1298 
1299   // Add original payload data.
1300   auto payload = packet.payload();
1301   memcpy(rtx_payload + kRtxHeaderSize, payload.data(), payload.size());
1302 
1303   return rtx_packet;
1304 }
1305 
RegisterRtpStatisticsCallback(StreamDataCountersCallback * callback)1306 void RTPSender::RegisterRtpStatisticsCallback(
1307     StreamDataCountersCallback* callback) {
1308   rtc::CritScope cs(&statistics_crit_);
1309   rtp_stats_callback_ = callback;
1310 }
1311 
GetRtpStatisticsCallback() const1312 StreamDataCountersCallback* RTPSender::GetRtpStatisticsCallback() const {
1313   rtc::CritScope cs(&statistics_crit_);
1314   return rtp_stats_callback_;
1315 }
1316 
BitrateSent() const1317 uint32_t RTPSender::BitrateSent() const {
1318   rtc::CritScope cs(&statistics_crit_);
1319   return total_bitrate_sent_.Rate(clock_->TimeInMilliseconds()).value_or(0);
1320 }
1321 
SetRtpState(const RtpState & rtp_state)1322 void RTPSender::SetRtpState(const RtpState& rtp_state) {
1323   rtc::CritScope lock(&send_critsect_);
1324   sequence_number_ = rtp_state.sequence_number;
1325   sequence_number_forced_ = true;
1326   timestamp_offset_ = rtp_state.start_timestamp;
1327   last_rtp_timestamp_ = rtp_state.timestamp;
1328   capture_time_ms_ = rtp_state.capture_time_ms;
1329   last_timestamp_time_ms_ = rtp_state.last_timestamp_time_ms;
1330   media_has_been_sent_ = rtp_state.media_has_been_sent;
1331 }
1332 
GetRtpState() const1333 RtpState RTPSender::GetRtpState() const {
1334   rtc::CritScope lock(&send_critsect_);
1335 
1336   RtpState state;
1337   state.sequence_number = sequence_number_;
1338   state.start_timestamp = timestamp_offset_;
1339   state.timestamp = last_rtp_timestamp_;
1340   state.capture_time_ms = capture_time_ms_;
1341   state.last_timestamp_time_ms = last_timestamp_time_ms_;
1342   state.media_has_been_sent = media_has_been_sent_;
1343 
1344   return state;
1345 }
1346 
SetRtxRtpState(const RtpState & rtp_state)1347 void RTPSender::SetRtxRtpState(const RtpState& rtp_state) {
1348   rtc::CritScope lock(&send_critsect_);
1349   sequence_number_rtx_ = rtp_state.sequence_number;
1350 }
1351 
GetRtxRtpState() const1352 RtpState RTPSender::GetRtxRtpState() const {
1353   rtc::CritScope lock(&send_critsect_);
1354 
1355   RtpState state;
1356   state.sequence_number = sequence_number_rtx_;
1357   state.start_timestamp = timestamp_offset_;
1358 
1359   return state;
1360 }
1361 
AddPacketToTransportFeedback(uint16_t packet_id,const RtpPacketToSend & packet,const PacedPacketInfo & pacing_info)1362 void RTPSender::AddPacketToTransportFeedback(
1363     uint16_t packet_id,
1364     const RtpPacketToSend& packet,
1365     const PacedPacketInfo& pacing_info) {
1366   size_t packet_size = packet.payload_size() + packet.padding_size();
1367   if (send_side_bwe_with_overhead_) {
1368     packet_size = packet.size();
1369   }
1370 
1371   if (transport_feedback_observer_) {
1372     transport_feedback_observer_->AddPacket(SSRC(), packet_id, packet_size,
1373                                             pacing_info);
1374   }
1375 }
1376 
UpdateRtpOverhead(const RtpPacketToSend & packet)1377 void RTPSender::UpdateRtpOverhead(const RtpPacketToSend& packet) {
1378   if (!overhead_observer_)
1379     return;
1380   size_t overhead_bytes_per_packet;
1381   {
1382     rtc::CritScope lock(&send_critsect_);
1383     if (rtp_overhead_bytes_per_packet_ == packet.headers_size()) {
1384       return;
1385     }
1386     rtp_overhead_bytes_per_packet_ = packet.headers_size();
1387     overhead_bytes_per_packet = rtp_overhead_bytes_per_packet_;
1388   }
1389   overhead_observer_->OnOverheadChanged(overhead_bytes_per_packet);
1390 }
1391 
LastTimestampTimeMs() const1392 int64_t RTPSender::LastTimestampTimeMs() const {
1393   rtc::CritScope lock(&send_critsect_);
1394   return last_timestamp_time_ms_;
1395 }
1396 
SendKeepAlive(uint8_t payload_type)1397 void RTPSender::SendKeepAlive(uint8_t payload_type) {
1398   std::unique_ptr<RtpPacketToSend> packet = AllocatePacket();
1399   packet->SetPayloadType(payload_type);
1400   // Set marker bit and timestamps in the same manner as plain padding packets.
1401   packet->SetMarker(false);
1402   {
1403     rtc::CritScope lock(&send_critsect_);
1404     packet->SetTimestamp(last_rtp_timestamp_);
1405     packet->set_capture_time_ms(capture_time_ms_);
1406   }
1407   AssignSequenceNumber(packet.get());
1408   SendToNetwork(std::move(packet), StorageType::kDontRetransmit,
1409                 RtpPacketSender::Priority::kLowPriority);
1410 }
1411 
1412 }  // namespace webrtc
1413