1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ 12 #define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ 13 14 #include <atomic> 15 #include <list> 16 #include <map> 17 #include <memory> 18 #include <string> 19 #include <vector> 20 21 #include "absl/types/optional.h" 22 #include "api/array_view.h" 23 #include "api/crypto/frame_decryptor_interface.h" 24 #include "api/video/color_space.h" 25 #include "api/video_codecs/video_codec.h" 26 #include "call/rtp_packet_sink_interface.h" 27 #include "call/syncable.h" 28 #include "call/video_receive_stream.h" 29 #include "modules/rtp_rtcp/include/receive_statistics.h" 30 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 31 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 32 #include "modules/rtp_rtcp/include/rtp_rtcp.h" 33 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 34 #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" 35 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" 36 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 37 #include "modules/rtp_rtcp/source/rtp_video_header.h" 38 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" 39 #include "modules/video_coding/h264_sps_pps_tracker.h" 40 #include "modules/video_coding/loss_notification_controller.h" 41 #include "modules/video_coding/packet_buffer.h" 42 #include "modules/video_coding/rtp_frame_reference_finder.h" 43 #include "modules/video_coding/unique_timestamp_counter.h" 44 #include "rtc_base/constructor_magic.h" 45 #include "rtc_base/experiments/field_trial_parser.h" 46 #include "rtc_base/numerics/sequence_number_util.h" 47 #include "rtc_base/synchronization/mutex.h" 48 #include "rtc_base/synchronization/sequence_checker.h" 49 #include "rtc_base/thread_annotations.h" 50 #include "rtc_base/thread_checker.h" 51 #include "video/buffered_frame_decryptor.h" 52 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" 53 54 namespace webrtc { 55 56 class DEPRECATED_NackModule; 57 class PacketRouter; 58 class ProcessThread; 59 class ReceiveStatistics; 60 class ReceiveStatisticsProxy; 61 class RtcpRttStats; 62 class RtpPacketReceived; 63 class Transport; 64 class UlpfecReceiver; 65 66 class RtpVideoStreamReceiver : public LossNotificationSender, 67 public RecoveredPacketReceiver, 68 public RtpPacketSinkInterface, 69 public KeyFrameRequestSender, 70 public video_coding::OnCompleteFrameCallback, 71 public OnDecryptedFrameCallback, 72 public OnDecryptionStatusChangeCallback, 73 public RtpVideoFrameReceiver { 74 public: 75 // DEPRECATED due to dependency on ReceiveStatisticsProxy. 76 RtpVideoStreamReceiver( 77 Clock* clock, 78 Transport* transport, 79 RtcpRttStats* rtt_stats, 80 // The packet router is optional; if provided, the RtpRtcp module for this 81 // stream is registered as a candidate for sending REMB and transport 82 // feedback. 83 PacketRouter* packet_router, 84 const VideoReceiveStream::Config* config, 85 ReceiveStatistics* rtp_receive_statistics, 86 ReceiveStatisticsProxy* receive_stats_proxy, 87 ProcessThread* process_thread, 88 NackSender* nack_sender, 89 // The KeyFrameRequestSender is optional; if not provided, key frame 90 // requests are sent via the internal RtpRtcp module. 91 KeyFrameRequestSender* keyframe_request_sender, 92 video_coding::OnCompleteFrameCallback* complete_frame_callback, 93 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, 94 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 95 96 RtpVideoStreamReceiver( 97 Clock* clock, 98 Transport* transport, 99 RtcpRttStats* rtt_stats, 100 // The packet router is optional; if provided, the RtpRtcp module for this 101 // stream is registered as a candidate for sending REMB and transport 102 // feedback. 103 PacketRouter* packet_router, 104 const VideoReceiveStream::Config* config, 105 ReceiveStatistics* rtp_receive_statistics, 106 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, 107 RtcpCnameCallback* rtcp_cname_callback, 108 ProcessThread* process_thread, 109 NackSender* nack_sender, 110 // The KeyFrameRequestSender is optional; if not provided, key frame 111 // requests are sent via the internal RtpRtcp module. 112 KeyFrameRequestSender* keyframe_request_sender, 113 video_coding::OnCompleteFrameCallback* complete_frame_callback, 114 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, 115 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 116 ~RtpVideoStreamReceiver() override; 117 118 void AddReceiveCodec(uint8_t payload_type, 119 const VideoCodec& video_codec, 120 const std::map<std::string, std::string>& codec_params, 121 bool raw_payload); 122 rtp_rtcp()123 RtpRtcp* rtp_rtcp() const { return rtp_rtcp_.get(); } 124 125 void StartReceive(); 126 void StopReceive(); 127 128 // Produces the transport-related timestamps; current_delay_ms is left unset. 129 absl::optional<Syncable::Info> GetSyncInfo() const; 130 131 bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); 132 133 void FrameContinuous(int64_t seq_num); 134 135 void FrameDecoded(int64_t seq_num); 136 137 void SignalNetworkState(NetworkState state); 138 139 // Returns number of different frames seen. GetUniqueFramesSeen()140 int GetUniqueFramesSeen() const { 141 RTC_DCHECK_RUN_ON(&worker_task_checker_); 142 return frame_counter_.GetUniqueSeen(); 143 } 144 145 // Implements RtpPacketSinkInterface. 146 void OnRtpPacket(const RtpPacketReceived& packet) override; 147 148 // Public only for tests. 149 void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, 150 const RtpPacketReceived& rtp_packet, 151 const RTPVideoHeader& video); 152 153 // Implements RecoveredPacketReceiver. 154 void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; 155 156 // Send an RTCP keyframe request. 157 void RequestKeyFrame() override; 158 159 // Implements LossNotificationSender. 160 void SendLossNotification(uint16_t last_decoded_seq_num, 161 uint16_t last_received_seq_num, 162 bool decodability_flag, 163 bool buffering_allowed) override; 164 165 bool IsUlpfecEnabled() const; 166 bool IsRetransmissionsEnabled() const; 167 168 // Returns true if a decryptor is attached and frames can be decrypted. 169 // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame 170 // Decryption not SRTP. 171 bool IsDecryptable() const; 172 173 // Don't use, still experimental. 174 void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers); 175 176 // Implements OnCompleteFrameCallback. 177 void OnCompleteFrame( 178 std::unique_ptr<video_coding::EncodedFrame> frame) override; 179 180 // Implements OnDecryptedFrameCallback. 181 void OnDecryptedFrame( 182 std::unique_ptr<video_coding::RtpFrameObject> frame) override; 183 184 // Implements OnDecryptionStatusChangeCallback. 185 void OnDecryptionStatusChange( 186 FrameDecryptorInterface::Status status) override; 187 188 // Optionally set a frame decryptor after a stream has started. This will not 189 // reset the decoder state. 190 void SetFrameDecryptor( 191 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); 192 193 // Sets a frame transformer after a stream has started, if no transformer 194 // has previously been set. Does not reset the decoder state. 195 void SetDepacketizerToDecoderFrameTransformer( 196 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 197 198 // Called by VideoReceiveStream when stats are updated. 199 void UpdateRtt(int64_t max_rtt_ms); 200 201 absl::optional<int64_t> LastReceivedPacketMs() const; 202 absl::optional<int64_t> LastReceivedKeyframePacketMs() const; 203 204 // RtpDemuxer only forwards a given RTP packet to one sink. However, some 205 // sinks, such as FlexFEC, might wish to be informed of all of the packets 206 // a given sink receives (or any set of sinks). They may do so by registering 207 // themselves as secondary sinks. 208 void AddSecondarySink(RtpPacketSinkInterface* sink); 209 void RemoveSecondarySink(const RtpPacketSinkInterface* sink); 210 211 private: 212 // Implements RtpVideoFrameReceiver. 213 void ManageFrame( 214 std::unique_ptr<video_coding::RtpFrameObject> frame) override; 215 216 // Used for buffering RTCP feedback messages and sending them all together. 217 // Note: 218 // 1. Key frame requests and NACKs are mutually exclusive, with the 219 // former taking precedence over the latter. 220 // 2. Loss notifications are orthogonal to either. (That is, may be sent 221 // alongside either.) 222 class RtcpFeedbackBuffer : public KeyFrameRequestSender, 223 public NackSender, 224 public LossNotificationSender { 225 public: 226 RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, 227 NackSender* nack_sender, 228 LossNotificationSender* loss_notification_sender); 229 230 ~RtcpFeedbackBuffer() override = default; 231 232 // KeyFrameRequestSender implementation. 233 void RequestKeyFrame() RTC_LOCKS_EXCLUDED(mutex_) override; 234 235 // NackSender implementation. 236 void SendNack(const std::vector<uint16_t>& sequence_numbers, 237 bool buffering_allowed) RTC_LOCKS_EXCLUDED(mutex_) override; 238 239 // LossNotificationSender implementation. 240 void SendLossNotification(uint16_t last_decoded_seq_num, 241 uint16_t last_received_seq_num, 242 bool decodability_flag, 243 bool buffering_allowed) 244 RTC_LOCKS_EXCLUDED(mutex_) override; 245 246 // Send all RTCP feedback messages buffered thus far. 247 void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); 248 249 private: 250 // LNTF-related state. 251 struct LossNotificationState { LossNotificationStateLossNotificationState252 LossNotificationState(uint16_t last_decoded_seq_num, 253 uint16_t last_received_seq_num, 254 bool decodability_flag) 255 : last_decoded_seq_num(last_decoded_seq_num), 256 last_received_seq_num(last_received_seq_num), 257 decodability_flag(decodability_flag) {} 258 259 uint16_t last_decoded_seq_num; 260 uint16_t last_received_seq_num; 261 bool decodability_flag; 262 }; 263 struct ConsumedRtcpFeedback { 264 bool request_key_frame = false; 265 std::vector<uint16_t> nack_sequence_numbers; 266 absl::optional<LossNotificationState> lntf_state; 267 }; 268 269 ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_); 270 ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked() 271 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 272 // This method is called both with and without mutex_ held. 273 void SendRtcpFeedback(ConsumedRtcpFeedback feedback); 274 275 KeyFrameRequestSender* const key_frame_request_sender_; 276 NackSender* const nack_sender_; 277 LossNotificationSender* const loss_notification_sender_; 278 279 // NACKs are accessible from two threads due to nack_module_ being a module. 280 Mutex mutex_; 281 282 // Key-frame-request-related state. 283 bool request_key_frame_ RTC_GUARDED_BY(mutex_); 284 285 // NACK-related state. 286 std::vector<uint16_t> nack_sequence_numbers_ RTC_GUARDED_BY(mutex_); 287 288 absl::optional<LossNotificationState> lntf_state_ RTC_GUARDED_BY(mutex_); 289 }; 290 enum ParseGenericDependenciesResult { 291 kDropPacket, 292 kHasGenericDescriptor, 293 kNoGenericDescriptor 294 }; 295 296 // Entry point doing non-stats work for a received packet. Called 297 // for the same packet both before and after RED decapsulation. 298 void ReceivePacket(const RtpPacketReceived& packet); 299 // Parses and handles RED headers. 300 // This function assumes that it's being called from only one thread. 301 void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); 302 void NotifyReceiverOfEmptyPacket(uint16_t seq_num); 303 void UpdateHistograms(); 304 bool IsRedEnabled() const; 305 void InsertSpsPpsIntoTracker(uint8_t payload_type); 306 void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); 307 ParseGenericDependenciesResult ParseGenericDependenciesExtension( 308 const RtpPacketReceived& rtp_packet, 309 RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); 310 void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame); 311 312 Clock* const clock_; 313 // Ownership of this object lies with VideoReceiveStream, which owns |this|. 314 const VideoReceiveStream::Config& config_; 315 PacketRouter* const packet_router_; 316 ProcessThread* const process_thread_; 317 318 RemoteNtpTimeEstimator ntp_estimator_; 319 320 RtpHeaderExtensionMap rtp_header_extensions_; 321 // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout 322 // delay that is specified in the received packets. 323 FieldTrialOptional<int> forced_playout_delay_max_ms_; 324 FieldTrialOptional<int> forced_playout_delay_min_ms_; 325 ReceiveStatistics* const rtp_receive_statistics_; 326 std::unique_ptr<UlpfecReceiver> ulpfec_receiver_; 327 328 SequenceChecker worker_task_checker_; 329 bool receiving_ RTC_GUARDED_BY(worker_task_checker_); 330 int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); 331 332 const std::unique_ptr<RtpRtcp> rtp_rtcp_; 333 334 video_coding::OnCompleteFrameCallback* complete_frame_callback_; 335 KeyFrameRequestSender* const keyframe_request_sender_; 336 337 RtcpFeedbackBuffer rtcp_feedback_buffer_; 338 std::unique_ptr<DEPRECATED_NackModule> nack_module_; 339 std::unique_ptr<LossNotificationController> loss_notification_controller_; 340 341 video_coding::PacketBuffer packet_buffer_; 342 UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); 343 SeqNumUnwrapper<uint16_t> frame_id_unwrapper_ 344 RTC_GUARDED_BY(worker_task_checker_); 345 346 // Video structure provided in the dependency descriptor in a first packet 347 // of a key frame. It is required to parse dependency descriptor in the 348 // following delta packets. 349 std::unique_ptr<FrameDependencyStructure> video_structure_ 350 RTC_GUARDED_BY(worker_task_checker_); 351 // Frame id of the last frame with the attached video structure. 352 // absl::nullopt when `video_structure_ == nullptr`; 353 absl::optional<int64_t> video_structure_frame_id_ 354 RTC_GUARDED_BY(worker_task_checker_); 355 356 Mutex reference_finder_lock_; 357 std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_ 358 RTC_GUARDED_BY(reference_finder_lock_); 359 absl::optional<VideoCodecType> current_codec_; 360 uint32_t last_assembled_frame_rtp_timestamp_; 361 362 Mutex last_seq_num_mutex_; 363 std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ 364 RTC_GUARDED_BY(last_seq_num_mutex_); 365 video_coding::H264SpsPpsTracker tracker_; 366 367 // Maps payload id to the depacketizer. 368 std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_; 369 370 // TODO(johan): Remove pt_codec_params_ once 371 // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. 372 // Maps a payload type to a map of out-of-band supplied codec parameters. 373 std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_; 374 int16_t last_payload_type_ = -1; 375 376 bool has_received_frame_; 377 378 std::vector<RtpPacketSinkInterface*> secondary_sinks_ 379 RTC_GUARDED_BY(worker_task_checker_); 380 381 // Info for GetSyncInfo is updated on network or worker thread, and queried on 382 // the worker thread. 383 mutable Mutex sync_info_lock_; 384 absl::optional<uint32_t> last_received_rtp_timestamp_ 385 RTC_GUARDED_BY(sync_info_lock_); 386 absl::optional<int64_t> last_received_rtp_system_time_ms_ 387 RTC_GUARDED_BY(sync_info_lock_); 388 389 // Used to validate the buffered frame decryptor is always run on the correct 390 // thread. 391 rtc::ThreadChecker network_tc_; 392 // Handles incoming encrypted frames and forwards them to the 393 // rtp_reference_finder if they are decryptable. 394 std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_ 395 RTC_PT_GUARDED_BY(network_tc_); 396 std::atomic<bool> frames_decryptable_; 397 absl::optional<ColorSpace> last_color_space_; 398 399 AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ 400 RTC_GUARDED_BY(worker_task_checker_); 401 402 int64_t last_completed_picture_id_ = 0; 403 404 rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> 405 frame_transformer_delegate_; 406 }; 407 408 } // namespace webrtc 409 410 #endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ 411