1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_video_stream_receiver.h"
12
13 #include <algorithm>
14 #include <limits>
15 #include <memory>
16 #include <utility>
17 #include <vector>
18
19 #include "absl/algorithm/container.h"
20 #include "absl/base/macros.h"
21 #include "absl/memory/memory.h"
22 #include "absl/types/optional.h"
23 #include "media/base/media_constants.h"
24 #include "modules/pacing/packet_router.h"
25 #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
26 #include "modules/rtp_rtcp/include/receive_statistics.h"
27 #include "modules/rtp_rtcp/include/rtp_cvo.h"
28 #include "modules/rtp_rtcp/include/ulpfec_receiver.h"
29 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
30 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
31 #include "modules/rtp_rtcp/source/rtp_format.h"
32 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
33 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
34 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
35 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
36 #include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
37 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
38 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
39 #include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h"
40 #include "modules/utility/include/process_thread.h"
41 #include "modules/video_coding/deprecated/nack_module.h"
42 #include "modules/video_coding/frame_object.h"
43 #include "modules/video_coding/h264_sprop_parameter_sets.h"
44 #include "modules/video_coding/h264_sps_pps_tracker.h"
45 #include "modules/video_coding/packet_buffer.h"
46 #include "rtc_base/checks.h"
47 #include "rtc_base/location.h"
48 #include "rtc_base/logging.h"
49 #include "rtc_base/strings/string_builder.h"
50 #include "system_wrappers/include/field_trial.h"
51 #include "system_wrappers/include/metrics.h"
52 #include "system_wrappers/include/ntp_time.h"
53 #include "video/receive_statistics_proxy.h"
54
55 namespace webrtc {
56
57 namespace {
58 // TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see:
59 // crbug.com/752886
60 constexpr int kPacketBufferStartSize = 512;
61 constexpr int kPacketBufferMaxSize = 2048;
62
PacketBufferMaxSize()63 int PacketBufferMaxSize() {
64 // The group here must be a positive power of 2, in which case that is used as
65 // size. All other values shall result in the default value being used.
66 const std::string group_name =
67 webrtc::field_trial::FindFullName("WebRTC-PacketBufferMaxSize");
68 int packet_buffer_max_size = kPacketBufferMaxSize;
69 if (!group_name.empty() &&
70 (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 ||
71 packet_buffer_max_size <= 0 ||
72 // Verify that the number is a positive power of 2.
73 (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) {
74 RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name;
75 packet_buffer_max_size = kPacketBufferMaxSize;
76 }
77 return packet_buffer_max_size;
78 }
79
CreateRtpRtcpModule(Clock * clock,ReceiveStatistics * receive_statistics,Transport * outgoing_transport,RtcpRttStats * rtt_stats,RtcpPacketTypeCounterObserver * rtcp_packet_type_counter_observer,RtcpCnameCallback * rtcp_cname_callback,uint32_t local_ssrc,RtcpEventObserver * rtcp_event_observer)80 std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
81 Clock* clock,
82 ReceiveStatistics* receive_statistics,
83 Transport* outgoing_transport,
84 RtcpRttStats* rtt_stats,
85 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
86 RtcpCnameCallback* rtcp_cname_callback,
87 uint32_t local_ssrc,
88 RtcpEventObserver* rtcp_event_observer) {
89 RtpRtcpInterface::Configuration configuration;
90 configuration.clock = clock;
91 configuration.audio = false;
92 configuration.receiver_only = true;
93 configuration.receive_statistics = receive_statistics;
94 configuration.outgoing_transport = outgoing_transport;
95 configuration.rtt_stats = rtt_stats;
96 configuration.rtcp_packet_type_counter_observer =
97 rtcp_packet_type_counter_observer;
98 configuration.rtcp_event_observer = rtcp_event_observer;
99 configuration.rtcp_cname_callback = rtcp_cname_callback;
100 configuration.local_media_ssrc = local_ssrc;
101
102 std::unique_ptr<RtpRtcp> rtp_rtcp = RtpRtcp::DEPRECATED_Create(configuration);
103 rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
104
105 return rtp_rtcp;
106 }
107
108 static const int kPacketLogIntervalMs = 10000;
109
110 } // namespace
111
RtcpFeedbackBuffer(KeyFrameRequestSender * key_frame_request_sender,NackSender * nack_sender,LossNotificationSender * loss_notification_sender)112 RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer(
113 KeyFrameRequestSender* key_frame_request_sender,
114 NackSender* nack_sender,
115 LossNotificationSender* loss_notification_sender)
116 : key_frame_request_sender_(key_frame_request_sender),
117 nack_sender_(nack_sender),
118 loss_notification_sender_(loss_notification_sender),
119 request_key_frame_(false) {
120 RTC_DCHECK(key_frame_request_sender_);
121 RTC_DCHECK(nack_sender_);
122 RTC_DCHECK(loss_notification_sender_);
123 }
124
RequestKeyFrame()125 void RtpVideoStreamReceiver::RtcpFeedbackBuffer::RequestKeyFrame() {
126 MutexLock lock(&mutex_);
127 request_key_frame_ = true;
128 }
129
SendNack(const std::vector<uint16_t> & sequence_numbers,bool buffering_allowed)130 void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendNack(
131 const std::vector<uint16_t>& sequence_numbers,
132 bool buffering_allowed) {
133 RTC_DCHECK(!sequence_numbers.empty());
134 MutexLock lock(&mutex_);
135 nack_sequence_numbers_.insert(nack_sequence_numbers_.end(),
136 sequence_numbers.cbegin(),
137 sequence_numbers.cend());
138 if (!buffering_allowed) {
139 // Note that while *buffering* is not allowed, *batching* is, meaning that
140 // previously buffered messages may be sent along with the current message.
141 SendRtcpFeedback(ConsumeRtcpFeedbackLocked());
142 }
143 }
144
SendLossNotification(uint16_t last_decoded_seq_num,uint16_t last_received_seq_num,bool decodability_flag,bool buffering_allowed)145 void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendLossNotification(
146 uint16_t last_decoded_seq_num,
147 uint16_t last_received_seq_num,
148 bool decodability_flag,
149 bool buffering_allowed) {
150 RTC_DCHECK(buffering_allowed);
151 MutexLock lock(&mutex_);
152 RTC_DCHECK(!lntf_state_)
153 << "SendLossNotification() called twice in a row with no call to "
154 "SendBufferedRtcpFeedback() in between.";
155 lntf_state_ = absl::make_optional<LossNotificationState>(
156 last_decoded_seq_num, last_received_seq_num, decodability_flag);
157 }
158
SendBufferedRtcpFeedback()159 void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() {
160 SendRtcpFeedback(ConsumeRtcpFeedback());
161 }
162
163 RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback
ConsumeRtcpFeedback()164 RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedback() {
165 MutexLock lock(&mutex_);
166 return ConsumeRtcpFeedbackLocked();
167 }
168
169 RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumedRtcpFeedback
ConsumeRtcpFeedbackLocked()170 RtpVideoStreamReceiver::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() {
171 ConsumedRtcpFeedback feedback;
172 std::swap(feedback.request_key_frame, request_key_frame_);
173 std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_);
174 std::swap(feedback.lntf_state, lntf_state_);
175 return feedback;
176 }
177
SendRtcpFeedback(ConsumedRtcpFeedback feedback)178 void RtpVideoStreamReceiver::RtcpFeedbackBuffer::SendRtcpFeedback(
179 ConsumedRtcpFeedback feedback) {
180 if (feedback.lntf_state) {
181 // If either a NACK or a key frame request is sent, we should buffer
182 // the LNTF and wait for them (NACK or key frame request) to trigger
183 // the compound feedback message.
184 // Otherwise, the LNTF should be sent out immediately.
185 const bool buffering_allowed =
186 feedback.request_key_frame || !feedback.nack_sequence_numbers.empty();
187
188 loss_notification_sender_->SendLossNotification(
189 feedback.lntf_state->last_decoded_seq_num,
190 feedback.lntf_state->last_received_seq_num,
191 feedback.lntf_state->decodability_flag, buffering_allowed);
192 }
193
194 if (feedback.request_key_frame) {
195 key_frame_request_sender_->RequestKeyFrame();
196 } else if (!feedback.nack_sequence_numbers.empty()) {
197 nack_sender_->SendNack(feedback.nack_sequence_numbers, true);
198 }
199 }
200
201 // DEPRECATED
RtpVideoStreamReceiver(Clock * clock,Transport * transport,RtcpRttStats * rtt_stats,PacketRouter * packet_router,const VideoReceiveStream::Config * config,ReceiveStatistics * rtp_receive_statistics,ReceiveStatisticsProxy * receive_stats_proxy,ProcessThread * process_thread,NackSender * nack_sender,KeyFrameRequestSender * keyframe_request_sender,video_coding::OnCompleteFrameCallback * complete_frame_callback,rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)202 RtpVideoStreamReceiver::RtpVideoStreamReceiver(
203 Clock* clock,
204 Transport* transport,
205 RtcpRttStats* rtt_stats,
206 PacketRouter* packet_router,
207 const VideoReceiveStream::Config* config,
208 ReceiveStatistics* rtp_receive_statistics,
209 ReceiveStatisticsProxy* receive_stats_proxy,
210 ProcessThread* process_thread,
211 NackSender* nack_sender,
212 KeyFrameRequestSender* keyframe_request_sender,
213 video_coding::OnCompleteFrameCallback* complete_frame_callback,
214 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
215 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
216 : RtpVideoStreamReceiver(clock,
217 transport,
218 rtt_stats,
219 packet_router,
220 config,
221 rtp_receive_statistics,
222 receive_stats_proxy,
223 receive_stats_proxy,
224 process_thread,
225 nack_sender,
226 keyframe_request_sender,
227 complete_frame_callback,
228 frame_decryptor,
229 frame_transformer) {}
230
RtpVideoStreamReceiver(Clock * clock,Transport * transport,RtcpRttStats * rtt_stats,PacketRouter * packet_router,const VideoReceiveStream::Config * config,ReceiveStatistics * rtp_receive_statistics,RtcpPacketTypeCounterObserver * rtcp_packet_type_counter_observer,RtcpCnameCallback * rtcp_cname_callback,ProcessThread * process_thread,NackSender * nack_sender,KeyFrameRequestSender * keyframe_request_sender,video_coding::OnCompleteFrameCallback * complete_frame_callback,rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)231 RtpVideoStreamReceiver::RtpVideoStreamReceiver(
232 Clock* clock,
233 Transport* transport,
234 RtcpRttStats* rtt_stats,
235 PacketRouter* packet_router,
236 const VideoReceiveStream::Config* config,
237 ReceiveStatistics* rtp_receive_statistics,
238 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
239 RtcpCnameCallback* rtcp_cname_callback,
240 ProcessThread* process_thread,
241 NackSender* nack_sender,
242 KeyFrameRequestSender* keyframe_request_sender,
243 video_coding::OnCompleteFrameCallback* complete_frame_callback,
244 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
245 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
246 : clock_(clock),
247 config_(*config),
248 packet_router_(packet_router),
249 process_thread_(process_thread),
250 ntp_estimator_(clock),
251 rtp_header_extensions_(config_.rtp.extensions),
252 forced_playout_delay_max_ms_("max_ms", absl::nullopt),
253 forced_playout_delay_min_ms_("min_ms", absl::nullopt),
254 rtp_receive_statistics_(rtp_receive_statistics),
255 ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc,
256 this,
257 config->rtp.extensions)),
258 receiving_(false),
259 last_packet_log_ms_(-1),
260 rtp_rtcp_(CreateRtpRtcpModule(clock,
261 rtp_receive_statistics_,
262 transport,
263 rtt_stats,
264 rtcp_packet_type_counter_observer,
265 rtcp_cname_callback,
266 config_.rtp.local_ssrc,
267 config_.rtp.rtcp_event_observer)),
268 complete_frame_callback_(complete_frame_callback),
269 keyframe_request_sender_(keyframe_request_sender),
270 // TODO(bugs.webrtc.org/10336): Let |rtcp_feedback_buffer_| communicate
271 // directly with |rtp_rtcp_|.
272 rtcp_feedback_buffer_(this, nack_sender, this),
273 packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()),
274 has_received_frame_(false),
275 frames_decryptable_(false),
276 absolute_capture_time_receiver_(clock) {
277 constexpr bool remb_candidate = true;
278 if (packet_router_)
279 packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
280
281 RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
282 << "A stream should not be configured with RTCP disabled. This value is "
283 "reserved for internal usage.";
284 // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
285 RTC_DCHECK(config_.rtp.local_ssrc != 0);
286 RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
287
288 rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode);
289 rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc);
290
291 static const int kMaxPacketAgeToNack = 450;
292 const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0)
293 ? kMaxPacketAgeToNack
294 : kDefaultMaxReorderingThreshold;
295 rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc,
296 max_reordering_threshold);
297 // TODO(nisse): For historic reasons, we applied the above
298 // max_reordering_threshold also for RTX stats, which makes little sense since
299 // we don't NACK rtx packets. Consider deleting the below block, and rely on
300 // the default threshold.
301 if (config_.rtp.rtx_ssrc) {
302 rtp_receive_statistics_->SetMaxReorderingThreshold(
303 config_.rtp.rtx_ssrc, max_reordering_threshold);
304 }
305
306 if (config_.rtp.rtcp_xr.receiver_reference_time_report)
307 rtp_rtcp_->SetRtcpXrRrtrStatus(true);
308
309 ParseFieldTrial(
310 {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_},
311 field_trial::FindFullName("WebRTC-ForcePlayoutDelay"));
312
313 process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE);
314
315 if (config_.rtp.lntf.enabled) {
316 loss_notification_controller_ =
317 std::make_unique<LossNotificationController>(&rtcp_feedback_buffer_,
318 &rtcp_feedback_buffer_);
319 }
320
321 if (config_.rtp.nack.rtp_history_ms != 0) {
322 nack_module_ = std::make_unique<DEPRECATED_NackModule>(
323 clock_, &rtcp_feedback_buffer_, &rtcp_feedback_buffer_);
324 process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE);
325 }
326
327 reference_finder_ =
328 std::make_unique<video_coding::RtpFrameReferenceFinder>(this);
329
330 // Only construct the encrypted receiver if frame encryption is enabled.
331 if (config_.crypto_options.sframe.require_frame_encryption) {
332 buffered_frame_decryptor_ =
333 std::make_unique<BufferedFrameDecryptor>(this, this);
334 if (frame_decryptor != nullptr) {
335 buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
336 }
337 }
338
339 if (frame_transformer) {
340 frame_transformer_delegate_ = new rtc::RefCountedObject<
341 RtpVideoStreamReceiverFrameTransformerDelegate>(
342 this, std::move(frame_transformer), rtc::Thread::Current(),
343 config_.rtp.remote_ssrc);
344 frame_transformer_delegate_->Init();
345 }
346 }
347
~RtpVideoStreamReceiver()348 RtpVideoStreamReceiver::~RtpVideoStreamReceiver() {
349 RTC_DCHECK(secondary_sinks_.empty());
350
351 if (nack_module_) {
352 process_thread_->DeRegisterModule(nack_module_.get());
353 }
354
355 process_thread_->DeRegisterModule(rtp_rtcp_.get());
356
357 if (packet_router_)
358 packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
359 UpdateHistograms();
360 if (frame_transformer_delegate_)
361 frame_transformer_delegate_->Reset();
362 }
363
AddReceiveCodec(uint8_t payload_type,const VideoCodec & video_codec,const std::map<std::string,std::string> & codec_params,bool raw_payload)364 void RtpVideoStreamReceiver::AddReceiveCodec(
365 uint8_t payload_type,
366 const VideoCodec& video_codec,
367 const std::map<std::string, std::string>& codec_params,
368 bool raw_payload) {
369 if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) ||
370 field_trial::IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) {
371 packet_buffer_.ForceSpsPpsIdrIsH264Keyframe();
372 }
373 payload_type_map_.emplace(
374 payload_type, raw_payload
375 ? std::make_unique<VideoRtpDepacketizerRaw>()
376 : CreateVideoRtpDepacketizer(video_codec.codecType));
377 pt_codec_params_.emplace(payload_type, codec_params);
378 }
379
GetSyncInfo() const380 absl::optional<Syncable::Info> RtpVideoStreamReceiver::GetSyncInfo() const {
381 Syncable::Info info;
382 if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs,
383 &info.capture_time_ntp_frac, nullptr, nullptr,
384 &info.capture_time_source_clock) != 0) {
385 return absl::nullopt;
386 }
387 {
388 MutexLock lock(&sync_info_lock_);
389 if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) {
390 return absl::nullopt;
391 }
392 info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
393 info.latest_receive_time_ms = *last_received_rtp_system_time_ms_;
394 }
395
396 // Leaves info.current_delay_ms uninitialized.
397 return info;
398 }
399
400 RtpVideoStreamReceiver::ParseGenericDependenciesResult
ParseGenericDependenciesExtension(const RtpPacketReceived & rtp_packet,RTPVideoHeader * video_header)401 RtpVideoStreamReceiver::ParseGenericDependenciesExtension(
402 const RtpPacketReceived& rtp_packet,
403 RTPVideoHeader* video_header) {
404 if (rtp_packet.HasExtension<RtpDependencyDescriptorExtension>()) {
405 webrtc::DependencyDescriptor dependency_descriptor;
406 if (!rtp_packet.GetExtension<RtpDependencyDescriptorExtension>(
407 video_structure_.get(), &dependency_descriptor)) {
408 // Descriptor is there, but failed to parse. Either it is invalid,
409 // or too old packet (after relevant video_structure_ changed),
410 // or too new packet (before relevant video_structure_ arrived).
411 // Drop such packet to be on the safe side.
412 // TODO(bugs.webrtc.org/10342): Stash too new packet.
413 RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
414 << " Failed to parse dependency descriptor.";
415 return kDropPacket;
416 }
417 if (dependency_descriptor.attached_structure != nullptr &&
418 !dependency_descriptor.first_packet_in_frame) {
419 RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc()
420 << "Invalid dependency descriptor: structure "
421 "attached to non first packet of a frame.";
422 return kDropPacket;
423 }
424 video_header->is_first_packet_in_frame =
425 dependency_descriptor.first_packet_in_frame;
426 video_header->is_last_packet_in_frame =
427 dependency_descriptor.last_packet_in_frame;
428
429 int64_t frame_id =
430 frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number);
431 auto& generic_descriptor_info = video_header->generic.emplace();
432 generic_descriptor_info.frame_id = frame_id;
433 generic_descriptor_info.spatial_index =
434 dependency_descriptor.frame_dependencies.spatial_id;
435 generic_descriptor_info.temporal_index =
436 dependency_descriptor.frame_dependencies.temporal_id;
437 for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) {
438 generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
439 }
440 generic_descriptor_info.decode_target_indications =
441 dependency_descriptor.frame_dependencies.decode_target_indications;
442 if (dependency_descriptor.resolution) {
443 video_header->width = dependency_descriptor.resolution->Width();
444 video_header->height = dependency_descriptor.resolution->Height();
445 }
446
447 // FrameDependencyStructure is sent in dependency descriptor of the first
448 // packet of a key frame and required for parsed dependency descriptor in
449 // all the following packets until next key frame.
450 // Save it if there is a (potentially) new structure.
451 if (dependency_descriptor.attached_structure) {
452 RTC_DCHECK(dependency_descriptor.first_packet_in_frame);
453 if (video_structure_frame_id_ > frame_id) {
454 RTC_LOG(LS_WARNING)
455 << "Arrived key frame with id " << frame_id << " and structure id "
456 << dependency_descriptor.attached_structure->structure_id
457 << " is older than the latest received key frame with id "
458 << *video_structure_frame_id_ << " and structure id "
459 << video_structure_->structure_id;
460 return kDropPacket;
461 }
462 video_structure_ = std::move(dependency_descriptor.attached_structure);
463 video_structure_frame_id_ = frame_id;
464 video_header->frame_type = VideoFrameType::kVideoFrameKey;
465 } else {
466 video_header->frame_type = VideoFrameType::kVideoFrameDelta;
467 }
468 return kHasGenericDescriptor;
469 }
470
471 RtpGenericFrameDescriptor generic_frame_descriptor;
472 if (!rtp_packet.GetExtension<RtpGenericFrameDescriptorExtension00>(
473 &generic_frame_descriptor)) {
474 return kNoGenericDescriptor;
475 }
476
477 video_header->is_first_packet_in_frame =
478 generic_frame_descriptor.FirstPacketInSubFrame();
479 video_header->is_last_packet_in_frame =
480 generic_frame_descriptor.LastPacketInSubFrame();
481
482 if (generic_frame_descriptor.FirstPacketInSubFrame()) {
483 video_header->frame_type =
484 generic_frame_descriptor.FrameDependenciesDiffs().empty()
485 ? VideoFrameType::kVideoFrameKey
486 : VideoFrameType::kVideoFrameDelta;
487
488 auto& generic_descriptor_info = video_header->generic.emplace();
489 int64_t frame_id =
490 frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId());
491 generic_descriptor_info.frame_id = frame_id;
492 generic_descriptor_info.spatial_index =
493 generic_frame_descriptor.SpatialLayer();
494 generic_descriptor_info.temporal_index =
495 generic_frame_descriptor.TemporalLayer();
496 for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) {
497 generic_descriptor_info.dependencies.push_back(frame_id - fdiff);
498 }
499 }
500 video_header->width = generic_frame_descriptor.Width();
501 video_header->height = generic_frame_descriptor.Height();
502 return kHasGenericDescriptor;
503 }
504
OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,const RtpPacketReceived & rtp_packet,const RTPVideoHeader & video)505 void RtpVideoStreamReceiver::OnReceivedPayloadData(
506 rtc::CopyOnWriteBuffer codec_payload,
507 const RtpPacketReceived& rtp_packet,
508 const RTPVideoHeader& video) {
509 RTC_DCHECK_RUN_ON(&worker_task_checker_);
510 auto packet = std::make_unique<video_coding::PacketBuffer::Packet>(
511 rtp_packet, video, ntp_estimator_.Estimate(rtp_packet.Timestamp()),
512 clock_->TimeInMilliseconds());
513
514 // Try to extrapolate absolute capture time if it is missing.
515 packet->packet_info.set_absolute_capture_time(
516 absolute_capture_time_receiver_.OnReceivePacket(
517 AbsoluteCaptureTimeReceiver::GetSource(packet->packet_info.ssrc(),
518 packet->packet_info.csrcs()),
519 packet->packet_info.rtp_timestamp(),
520 // Assume frequency is the same one for all video frames.
521 kVideoPayloadTypeFrequency,
522 packet->packet_info.absolute_capture_time()));
523
524 RTPVideoHeader& video_header = packet->video_header;
525 video_header.rotation = kVideoRotation_0;
526 video_header.content_type = VideoContentType::UNSPECIFIED;
527 video_header.video_timing.flags = VideoSendTiming::kInvalid;
528 video_header.is_last_packet_in_frame |= rtp_packet.Marker();
529
530 if (const auto* vp9_header =
531 absl::get_if<RTPVideoHeaderVP9>(&video_header.video_type_header)) {
532 video_header.is_last_packet_in_frame |= vp9_header->end_of_frame;
533 video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame;
534 }
535
536 rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation);
537 rtp_packet.GetExtension<VideoContentTypeExtension>(
538 &video_header.content_type);
539 rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
540 if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) {
541 video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_;
542 video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_;
543 } else {
544 rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
545 }
546
547 ParseGenericDependenciesResult generic_descriptor_state =
548 ParseGenericDependenciesExtension(rtp_packet, &video_header);
549 if (generic_descriptor_state == kDropPacket)
550 return;
551
552 // Color space should only be transmitted in the last packet of a frame,
553 // therefore, neglect it otherwise so that last_color_space_ is not reset by
554 // mistake.
555 if (video_header.is_last_packet_in_frame) {
556 video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>();
557 if (video_header.color_space ||
558 video_header.frame_type == VideoFrameType::kVideoFrameKey) {
559 // Store color space since it's only transmitted when changed or for key
560 // frames. Color space will be cleared if a key frame is transmitted
561 // without color space information.
562 last_color_space_ = video_header.color_space;
563 } else if (last_color_space_) {
564 video_header.color_space = last_color_space_;
565 }
566 }
567
568 if (loss_notification_controller_) {
569 if (rtp_packet.recovered()) {
570 // TODO(bugs.webrtc.org/10336): Implement support for reordering.
571 RTC_LOG(LS_INFO)
572 << "LossNotificationController does not support reordering.";
573 } else if (generic_descriptor_state == kNoGenericDescriptor) {
574 RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
575 "frame descriptor, but it is missing.";
576 } else {
577 if (video_header.is_first_packet_in_frame) {
578 RTC_DCHECK(video_header.generic);
579 LossNotificationController::FrameDetails frame;
580 frame.is_keyframe =
581 video_header.frame_type == VideoFrameType::kVideoFrameKey;
582 frame.frame_id = video_header.generic->frame_id;
583 frame.frame_dependencies = video_header.generic->dependencies;
584 loss_notification_controller_->OnReceivedPacket(
585 rtp_packet.SequenceNumber(), &frame);
586 } else {
587 loss_notification_controller_->OnReceivedPacket(
588 rtp_packet.SequenceNumber(), nullptr);
589 }
590 }
591 }
592
593 if (nack_module_) {
594 const bool is_keyframe =
595 video_header.is_first_packet_in_frame &&
596 video_header.frame_type == VideoFrameType::kVideoFrameKey;
597
598 packet->times_nacked = nack_module_->OnReceivedPacket(
599 rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered());
600 } else {
601 packet->times_nacked = -1;
602 }
603
604 if (codec_payload.size() == 0) {
605 NotifyReceiverOfEmptyPacket(packet->seq_num);
606 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
607 return;
608 }
609
610 if (packet->codec() == kVideoCodecH264) {
611 // Only when we start to receive packets will we know what payload type
612 // that will be used. When we know the payload type insert the correct
613 // sps/pps into the tracker.
614 if (packet->payload_type != last_payload_type_) {
615 last_payload_type_ = packet->payload_type;
616 InsertSpsPpsIntoTracker(packet->payload_type);
617 }
618
619 video_coding::H264SpsPpsTracker::FixedBitstream fixed =
620 tracker_.CopyAndFixBitstream(
621 rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()),
622 &packet->video_header);
623
624 switch (fixed.action) {
625 case video_coding::H264SpsPpsTracker::kRequestKeyframe:
626 rtcp_feedback_buffer_.RequestKeyFrame();
627 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
628 ABSL_FALLTHROUGH_INTENDED;
629 case video_coding::H264SpsPpsTracker::kDrop:
630 return;
631 case video_coding::H264SpsPpsTracker::kInsert:
632 packet->video_payload = std::move(fixed.bitstream);
633 break;
634 }
635
636 } else {
637 packet->video_payload = std::move(codec_payload);
638 }
639
640 rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
641 frame_counter_.Add(packet->timestamp);
642 OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet)));
643 }
644
OnRecoveredPacket(const uint8_t * rtp_packet,size_t rtp_packet_length)645 void RtpVideoStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
646 size_t rtp_packet_length) {
647 RtpPacketReceived packet;
648 if (!packet.Parse(rtp_packet, rtp_packet_length))
649 return;
650 if (packet.PayloadType() == config_.rtp.red_payload_type) {
651 RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation";
652 return;
653 }
654
655 packet.IdentifyExtensions(rtp_header_extensions_);
656 packet.set_payload_type_frequency(kVideoPayloadTypeFrequency);
657 // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both
658 // original (decapsulated) media packets and recovered packets to
659 // this callback. We need a way to distinguish, for setting
660 // packet.recovered() correctly. Ideally, move RED decapsulation out
661 // of the Ulpfec implementation.
662
663 ReceivePacket(packet);
664 }
665
666 // This method handles both regular RTP packets and packets recovered
667 // via FlexFEC.
OnRtpPacket(const RtpPacketReceived & packet)668 void RtpVideoStreamReceiver::OnRtpPacket(const RtpPacketReceived& packet) {
669 RTC_DCHECK_RUN_ON(&worker_task_checker_);
670
671 if (!receiving_) {
672 return;
673 }
674
675 if (!packet.recovered()) {
676 // TODO(nisse): Exclude out-of-order packets?
677 int64_t now_ms = clock_->TimeInMilliseconds();
678 {
679 MutexLock lock(&sync_info_lock_);
680 last_received_rtp_timestamp_ = packet.Timestamp();
681 last_received_rtp_system_time_ms_ = now_ms;
682 }
683 // Periodically log the RTP header of incoming packets.
684 if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) {
685 rtc::StringBuilder ss;
686 ss << "Packet received on SSRC: " << packet.Ssrc()
687 << " with payload type: " << static_cast<int>(packet.PayloadType())
688 << ", timestamp: " << packet.Timestamp()
689 << ", sequence number: " << packet.SequenceNumber()
690 << ", arrival time: " << packet.arrival_time_ms();
691 int32_t time_offset;
692 if (packet.GetExtension<TransmissionOffset>(&time_offset)) {
693 ss << ", toffset: " << time_offset;
694 }
695 uint32_t send_time;
696 if (packet.GetExtension<AbsoluteSendTime>(&send_time)) {
697 ss << ", abs send time: " << send_time;
698 }
699 RTC_LOG(LS_INFO) << ss.str();
700 last_packet_log_ms_ = now_ms;
701 }
702 }
703
704 ReceivePacket(packet);
705
706 // Update receive statistics after ReceivePacket.
707 // Receive statistics will be reset if the payload type changes (make sure
708 // that the first packet is included in the stats).
709 if (!packet.recovered()) {
710 rtp_receive_statistics_->OnRtpPacket(packet);
711 }
712
713 for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) {
714 secondary_sink->OnRtpPacket(packet);
715 }
716 }
717
RequestKeyFrame()718 void RtpVideoStreamReceiver::RequestKeyFrame() {
719 // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests
720 // issued by anything other than the LossNotificationController if it (the
721 // sender) is relying on LNTF alone.
722 if (keyframe_request_sender_) {
723 keyframe_request_sender_->RequestKeyFrame();
724 } else {
725 rtp_rtcp_->SendPictureLossIndication();
726 }
727 }
728
SendLossNotification(uint16_t last_decoded_seq_num,uint16_t last_received_seq_num,bool decodability_flag,bool buffering_allowed)729 void RtpVideoStreamReceiver::SendLossNotification(
730 uint16_t last_decoded_seq_num,
731 uint16_t last_received_seq_num,
732 bool decodability_flag,
733 bool buffering_allowed) {
734 RTC_DCHECK(config_.rtp.lntf.enabled);
735 rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num,
736 decodability_flag, buffering_allowed);
737 }
738
IsUlpfecEnabled() const739 bool RtpVideoStreamReceiver::IsUlpfecEnabled() const {
740 return config_.rtp.ulpfec_payload_type != -1;
741 }
742
IsRetransmissionsEnabled() const743 bool RtpVideoStreamReceiver::IsRetransmissionsEnabled() const {
744 return config_.rtp.nack.rtp_history_ms > 0;
745 }
746
RequestPacketRetransmit(const std::vector<uint16_t> & sequence_numbers)747 void RtpVideoStreamReceiver::RequestPacketRetransmit(
748 const std::vector<uint16_t>& sequence_numbers) {
749 rtp_rtcp_->SendNack(sequence_numbers);
750 }
751
IsDecryptable() const752 bool RtpVideoStreamReceiver::IsDecryptable() const {
753 return frames_decryptable_.load();
754 }
755
OnInsertedPacket(video_coding::PacketBuffer::InsertResult result)756 void RtpVideoStreamReceiver::OnInsertedPacket(
757 video_coding::PacketBuffer::InsertResult result) {
758 video_coding::PacketBuffer::Packet* first_packet = nullptr;
759 int max_nack_count;
760 int64_t min_recv_time;
761 int64_t max_recv_time;
762 std::vector<rtc::ArrayView<const uint8_t>> payloads;
763 RtpPacketInfos::vector_type packet_infos;
764
765 bool frame_boundary = true;
766 for (auto& packet : result.packets) {
767 // PacketBuffer promisses frame boundaries are correctly set on each
768 // packet. Document that assumption with the DCHECKs.
769 RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame());
770 if (packet->is_first_packet_in_frame()) {
771 first_packet = packet.get();
772 max_nack_count = packet->times_nacked;
773 min_recv_time = packet->packet_info.receive_time_ms();
774 max_recv_time = packet->packet_info.receive_time_ms();
775 payloads.clear();
776 packet_infos.clear();
777 } else {
778 max_nack_count = std::max(max_nack_count, packet->times_nacked);
779 min_recv_time =
780 std::min(min_recv_time, packet->packet_info.receive_time_ms());
781 max_recv_time =
782 std::max(max_recv_time, packet->packet_info.receive_time_ms());
783 }
784 payloads.emplace_back(packet->video_payload);
785 packet_infos.push_back(packet->packet_info);
786
787 frame_boundary = packet->is_last_packet_in_frame();
788 if (packet->is_last_packet_in_frame()) {
789 auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
790 RTC_CHECK(depacketizer_it != payload_type_map_.end());
791
792 rtc::scoped_refptr<EncodedImageBuffer> bitstream =
793 depacketizer_it->second->AssembleFrame(payloads);
794 if (!bitstream) {
795 // Failed to assemble a frame. Discard and continue.
796 continue;
797 }
798
799 const video_coding::PacketBuffer::Packet& last_packet = *packet;
800 OnAssembledFrame(std::make_unique<video_coding::RtpFrameObject>(
801 first_packet->seq_num, //
802 last_packet.seq_num, //
803 last_packet.marker_bit, //
804 max_nack_count, //
805 min_recv_time, //
806 max_recv_time, //
807 first_packet->timestamp, //
808 first_packet->ntp_time_ms, //
809 last_packet.video_header.video_timing, //
810 first_packet->payload_type, //
811 first_packet->codec(), //
812 last_packet.video_header.rotation, //
813 last_packet.video_header.content_type, //
814 first_packet->video_header, //
815 last_packet.video_header.color_space, //
816 RtpPacketInfos(std::move(packet_infos)), //
817 std::move(bitstream)));
818 }
819 }
820 RTC_DCHECK(frame_boundary);
821 if (result.buffer_cleared) {
822 RequestKeyFrame();
823 }
824 }
825
OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame)826 void RtpVideoStreamReceiver::OnAssembledFrame(
827 std::unique_ptr<video_coding::RtpFrameObject> frame) {
828 RTC_DCHECK_RUN_ON(&network_tc_);
829 RTC_DCHECK(frame);
830
831 const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& descriptor =
832 frame->GetRtpVideoHeader().generic;
833
834 if (loss_notification_controller_ && descriptor) {
835 loss_notification_controller_->OnAssembledFrame(
836 frame->first_seq_num(), descriptor->frame_id,
837 absl::c_linear_search(descriptor->decode_target_indications,
838 DecodeTargetIndication::kDiscardable),
839 descriptor->dependencies);
840 }
841
842 // If frames arrive before a key frame, they would not be decodable.
843 // In that case, request a key frame ASAP.
844 if (!has_received_frame_) {
845 if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
846 // |loss_notification_controller_|, if present, would have already
847 // requested a key frame when the first packet for the non-key frame
848 // had arrived, so no need to replicate the request.
849 if (!loss_notification_controller_) {
850 RequestKeyFrame();
851 }
852 }
853 has_received_frame_ = true;
854 }
855
856 MutexLock lock(&reference_finder_lock_);
857 // Reset |reference_finder_| if |frame| is new and the codec have changed.
858 if (current_codec_) {
859 bool frame_is_newer =
860 AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);
861
862 if (frame->codec_type() != current_codec_) {
863 if (frame_is_newer) {
864 // When we reset the |reference_finder_| we don't want new picture ids
865 // to overlap with old picture ids. To ensure that doesn't happen we
866 // start from the |last_completed_picture_id_| and add an offset in case
867 // of reordering.
868 reference_finder_ =
869 std::make_unique<video_coding::RtpFrameReferenceFinder>(
870 this, last_completed_picture_id_ +
871 std::numeric_limits<uint16_t>::max());
872 current_codec_ = frame->codec_type();
873 } else {
874 // Old frame from before the codec switch, discard it.
875 return;
876 }
877 }
878
879 if (frame_is_newer) {
880 last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
881 }
882 } else {
883 current_codec_ = frame->codec_type();
884 last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
885 }
886
887 if (buffered_frame_decryptor_ != nullptr) {
888 buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
889 } else if (frame_transformer_delegate_) {
890 frame_transformer_delegate_->TransformFrame(std::move(frame));
891 } else {
892 reference_finder_->ManageFrame(std::move(frame));
893 }
894 }
895
OnCompleteFrame(std::unique_ptr<video_coding::EncodedFrame> frame)896 void RtpVideoStreamReceiver::OnCompleteFrame(
897 std::unique_ptr<video_coding::EncodedFrame> frame) {
898 {
899 MutexLock lock(&last_seq_num_mutex_);
900 video_coding::RtpFrameObject* rtp_frame =
901 static_cast<video_coding::RtpFrameObject*>(frame.get());
902 last_seq_num_for_pic_id_[rtp_frame->id.picture_id] =
903 rtp_frame->last_seq_num();
904 }
905 last_completed_picture_id_ =
906 std::max(last_completed_picture_id_, frame->id.picture_id);
907 complete_frame_callback_->OnCompleteFrame(std::move(frame));
908 }
909
OnDecryptedFrame(std::unique_ptr<video_coding::RtpFrameObject> frame)910 void RtpVideoStreamReceiver::OnDecryptedFrame(
911 std::unique_ptr<video_coding::RtpFrameObject> frame) {
912 MutexLock lock(&reference_finder_lock_);
913 reference_finder_->ManageFrame(std::move(frame));
914 }
915
OnDecryptionStatusChange(FrameDecryptorInterface::Status status)916 void RtpVideoStreamReceiver::OnDecryptionStatusChange(
917 FrameDecryptorInterface::Status status) {
918 frames_decryptable_.store(
919 (status == FrameDecryptorInterface::Status::kOk) ||
920 (status == FrameDecryptorInterface::Status::kRecoverable));
921 }
922
SetFrameDecryptor(rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor)923 void RtpVideoStreamReceiver::SetFrameDecryptor(
924 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) {
925 RTC_DCHECK_RUN_ON(&network_tc_);
926 if (buffered_frame_decryptor_ == nullptr) {
927 buffered_frame_decryptor_ =
928 std::make_unique<BufferedFrameDecryptor>(this, this);
929 }
930 buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
931 }
932
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)933 void RtpVideoStreamReceiver::SetDepacketizerToDecoderFrameTransformer(
934 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
935 RTC_DCHECK_RUN_ON(&network_tc_);
936 frame_transformer_delegate_ =
937 new rtc::RefCountedObject<RtpVideoStreamReceiverFrameTransformerDelegate>(
938 this, std::move(frame_transformer), rtc::Thread::Current(),
939 config_.rtp.remote_ssrc);
940 frame_transformer_delegate_->Init();
941 }
942
UpdateRtt(int64_t max_rtt_ms)943 void RtpVideoStreamReceiver::UpdateRtt(int64_t max_rtt_ms) {
944 if (nack_module_)
945 nack_module_->UpdateRtt(max_rtt_ms);
946 }
947
LastReceivedPacketMs() const948 absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedPacketMs() const {
949 return packet_buffer_.LastReceivedPacketMs();
950 }
951
LastReceivedKeyframePacketMs() const952 absl::optional<int64_t> RtpVideoStreamReceiver::LastReceivedKeyframePacketMs()
953 const {
954 return packet_buffer_.LastReceivedKeyframePacketMs();
955 }
956
AddSecondarySink(RtpPacketSinkInterface * sink)957 void RtpVideoStreamReceiver::AddSecondarySink(RtpPacketSinkInterface* sink) {
958 RTC_DCHECK_RUN_ON(&worker_task_checker_);
959 RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink));
960 secondary_sinks_.push_back(sink);
961 }
962
RemoveSecondarySink(const RtpPacketSinkInterface * sink)963 void RtpVideoStreamReceiver::RemoveSecondarySink(
964 const RtpPacketSinkInterface* sink) {
965 RTC_DCHECK_RUN_ON(&worker_task_checker_);
966 auto it = absl::c_find(secondary_sinks_, sink);
967 if (it == secondary_sinks_.end()) {
968 // We might be rolling-back a call whose setup failed mid-way. In such a
969 // case, it's simpler to remove "everything" rather than remember what
970 // has already been added.
971 RTC_LOG(LS_WARNING) << "Removal of unknown sink.";
972 return;
973 }
974 secondary_sinks_.erase(it);
975 }
976
ManageFrame(std::unique_ptr<video_coding::RtpFrameObject> frame)977 void RtpVideoStreamReceiver::ManageFrame(
978 std::unique_ptr<video_coding::RtpFrameObject> frame) {
979 MutexLock lock(&reference_finder_lock_);
980 reference_finder_->ManageFrame(std::move(frame));
981 }
982
ReceivePacket(const RtpPacketReceived & packet)983 void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
984 if (packet.payload_size() == 0) {
985 // Padding or keep-alive packet.
986 // TODO(nisse): Could drop empty packets earlier, but need to figure out how
987 // they should be counted in stats.
988 NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
989 return;
990 }
991 if (packet.PayloadType() == config_.rtp.red_payload_type) {
992 ParseAndHandleEncapsulatingHeader(packet);
993 return;
994 }
995
996 const auto type_it = payload_type_map_.find(packet.PayloadType());
997 if (type_it == payload_type_map_.end()) {
998 return;
999 }
1000 absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
1001 type_it->second->Parse(packet.PayloadBuffer());
1002 if (parsed_payload == absl::nullopt) {
1003 RTC_LOG(LS_WARNING) << "Failed parsing payload.";
1004 return;
1005 }
1006
1007 OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet,
1008 parsed_payload->video_header);
1009 }
1010
ParseAndHandleEncapsulatingHeader(const RtpPacketReceived & packet)1011 void RtpVideoStreamReceiver::ParseAndHandleEncapsulatingHeader(
1012 const RtpPacketReceived& packet) {
1013 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1014 if (packet.PayloadType() == config_.rtp.red_payload_type &&
1015 packet.payload_size() > 0) {
1016 if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) {
1017 // Notify video_receiver about received FEC packets to avoid NACKing these
1018 // packets.
1019 NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
1020 }
1021 if (!ulpfec_receiver_->AddReceivedRedPacket(
1022 packet, config_.rtp.ulpfec_payload_type)) {
1023 return;
1024 }
1025 ulpfec_receiver_->ProcessReceivedFec();
1026 }
1027 }
1028
1029 // In the case of a video stream without picture ids and no rtx the
1030 // RtpFrameReferenceFinder will need to know about padding to
1031 // correctly calculate frame references.
NotifyReceiverOfEmptyPacket(uint16_t seq_num)1032 void RtpVideoStreamReceiver::NotifyReceiverOfEmptyPacket(uint16_t seq_num) {
1033 {
1034 MutexLock lock(&reference_finder_lock_);
1035 reference_finder_->PaddingReceived(seq_num);
1036 }
1037 OnInsertedPacket(packet_buffer_.InsertPadding(seq_num));
1038 if (nack_module_) {
1039 nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false,
1040 /* is _recovered = */ false);
1041 }
1042 if (loss_notification_controller_) {
1043 // TODO(bugs.webrtc.org/10336): Handle empty packets.
1044 RTC_LOG(LS_WARNING)
1045 << "LossNotificationController does not expect empty packets.";
1046 }
1047 }
1048
DeliverRtcp(const uint8_t * rtcp_packet,size_t rtcp_packet_length)1049 bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet,
1050 size_t rtcp_packet_length) {
1051 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1052
1053 if (!receiving_) {
1054 return false;
1055 }
1056
1057 rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length);
1058
1059 int64_t rtt = 0;
1060 rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr);
1061 if (rtt == 0) {
1062 // Waiting for valid rtt.
1063 return true;
1064 }
1065 uint32_t ntp_secs = 0;
1066 uint32_t ntp_frac = 0;
1067 uint32_t rtp_timestamp = 0;
1068 uint32_t recieved_ntp_secs = 0;
1069 uint32_t recieved_ntp_frac = 0;
1070 if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &recieved_ntp_secs,
1071 &recieved_ntp_frac, &rtp_timestamp) != 0) {
1072 // Waiting for RTCP.
1073 return true;
1074 }
1075 NtpTime recieved_ntp(recieved_ntp_secs, recieved_ntp_frac);
1076 int64_t time_since_recieved =
1077 clock_->CurrentNtpInMilliseconds() - recieved_ntp.ToMs();
1078 // Don't use old SRs to estimate time.
1079 if (time_since_recieved <= 1) {
1080 ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
1081 absl::optional<int64_t> remote_to_local_clock_offset_ms =
1082 ntp_estimator_.EstimateRemoteToLocalClockOffsetMs();
1083 if (remote_to_local_clock_offset_ms.has_value()) {
1084 absolute_capture_time_receiver_.SetRemoteToLocalClockOffset(
1085 Int64MsToQ32x32(*remote_to_local_clock_offset_ms));
1086 }
1087 }
1088
1089 return true;
1090 }
1091
FrameContinuous(int64_t picture_id)1092 void RtpVideoStreamReceiver::FrameContinuous(int64_t picture_id) {
1093 if (!nack_module_)
1094 return;
1095
1096 int seq_num = -1;
1097 {
1098 MutexLock lock(&last_seq_num_mutex_);
1099 auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
1100 if (seq_num_it != last_seq_num_for_pic_id_.end())
1101 seq_num = seq_num_it->second;
1102 }
1103 if (seq_num != -1)
1104 nack_module_->ClearUpTo(seq_num);
1105 }
1106
FrameDecoded(int64_t picture_id)1107 void RtpVideoStreamReceiver::FrameDecoded(int64_t picture_id) {
1108 int seq_num = -1;
1109 {
1110 MutexLock lock(&last_seq_num_mutex_);
1111 auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
1112 if (seq_num_it != last_seq_num_for_pic_id_.end()) {
1113 seq_num = seq_num_it->second;
1114 last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
1115 ++seq_num_it);
1116 }
1117 }
1118 if (seq_num != -1) {
1119 packet_buffer_.ClearTo(seq_num);
1120 MutexLock lock(&reference_finder_lock_);
1121 reference_finder_->ClearTo(seq_num);
1122 }
1123 }
1124
SignalNetworkState(NetworkState state)1125 void RtpVideoStreamReceiver::SignalNetworkState(NetworkState state) {
1126 rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
1127 : RtcpMode::kOff);
1128 }
1129
StartReceive()1130 void RtpVideoStreamReceiver::StartReceive() {
1131 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1132 receiving_ = true;
1133 }
1134
StopReceive()1135 void RtpVideoStreamReceiver::StopReceive() {
1136 RTC_DCHECK_RUN_ON(&worker_task_checker_);
1137 receiving_ = false;
1138 }
1139
UpdateHistograms()1140 void RtpVideoStreamReceiver::UpdateHistograms() {
1141 FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter();
1142 if (counter.first_packet_time_ms == -1)
1143 return;
1144
1145 int64_t elapsed_sec =
1146 (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000;
1147 if (elapsed_sec < metrics::kMinRunTimeInSeconds)
1148 return;
1149
1150 if (counter.num_packets > 0) {
1151 RTC_HISTOGRAM_PERCENTAGE(
1152 "WebRTC.Video.ReceivedFecPacketsInPercent",
1153 static_cast<int>(counter.num_fec_packets * 100 / counter.num_packets));
1154 }
1155 if (counter.num_fec_packets > 0) {
1156 RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec",
1157 static_cast<int>(counter.num_recovered_packets *
1158 100 / counter.num_fec_packets));
1159 }
1160 if (config_.rtp.ulpfec_payload_type != -1) {
1161 RTC_HISTOGRAM_COUNTS_10000(
1162 "WebRTC.Video.FecBitrateReceivedInKbps",
1163 static_cast<int>(counter.num_bytes * 8 / elapsed_sec / 1000));
1164 }
1165 }
1166
InsertSpsPpsIntoTracker(uint8_t payload_type)1167 void RtpVideoStreamReceiver::InsertSpsPpsIntoTracker(uint8_t payload_type) {
1168 auto codec_params_it = pt_codec_params_.find(payload_type);
1169 if (codec_params_it == pt_codec_params_.end())
1170 return;
1171
1172 RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for"
1173 " payload type: "
1174 << static_cast<int>(payload_type);
1175
1176 H264SpropParameterSets sprop_decoder;
1177 auto sprop_base64_it =
1178 codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets);
1179
1180 if (sprop_base64_it == codec_params_it->second.end())
1181 return;
1182
1183 if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str()))
1184 return;
1185
1186 tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(),
1187 sprop_decoder.pps_nalu());
1188 }
1189
1190 } // namespace webrtc
1191