1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 12 #define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/crypto/frame_decryptor_interface.h" 21 #include "api/video/color_space.h" 22 #include "api/video_codecs/video_codec.h" 23 #include "call/rtp_packet_sink_interface.h" 24 #include "call/syncable.h" 25 #include "call/video_receive_stream.h" 26 #include "modules/rtp_rtcp/include/receive_statistics.h" 27 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" 28 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" 29 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" 30 #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" 31 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" 32 #include "modules/rtp_rtcp/source/rtp_packet_received.h" 33 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" 34 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" 35 #include "modules/rtp_rtcp/source/rtp_video_header.h" 36 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" 37 #include "modules/video_coding/h264_sps_pps_tracker.h" 38 #include "modules/video_coding/loss_notification_controller.h" 39 #include "modules/video_coding/packet_buffer.h" 40 #include "modules/video_coding/rtp_frame_reference_finder.h" 41 #include "modules/video_coding/unique_timestamp_counter.h" 42 #include "rtc_base/constructor_magic.h" 43 #include "rtc_base/experiments/field_trial_parser.h" 44 #include "rtc_base/numerics/sequence_number_util.h" 45 #include "rtc_base/synchronization/sequence_checker.h" 46 #include "rtc_base/thread_annotations.h" 47 #include "video/buffered_frame_decryptor.h" 48 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" 49 50 namespace webrtc { 51 52 class NackModule2; 53 class PacketRouter; 54 class ProcessThread; 55 class ReceiveStatistics; 56 class RtcpRttStats; 57 class RtpPacketReceived; 58 class Transport; 59 class UlpfecReceiver; 60 61 class RtpVideoStreamReceiver2 : public LossNotificationSender, 62 public RecoveredPacketReceiver, 63 public RtpPacketSinkInterface, 64 public KeyFrameRequestSender, 65 public video_coding::OnCompleteFrameCallback, 66 public OnDecryptedFrameCallback, 67 public OnDecryptionStatusChangeCallback, 68 public RtpVideoFrameReceiver { 69 public: 70 RtpVideoStreamReceiver2( 71 TaskQueueBase* current_queue, 72 Clock* clock, 73 Transport* transport, 74 RtcpRttStats* rtt_stats, 75 // The packet router is optional; if provided, the RtpRtcp module for this 76 // stream is registered as a candidate for sending REMB and transport 77 // feedback. 78 PacketRouter* packet_router, 79 const VideoReceiveStream::Config* config, 80 ReceiveStatistics* rtp_receive_statistics, 81 RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, 82 RtcpCnameCallback* rtcp_cname_callback, 83 VCMReceiveStatisticsCallback* vcm_receive_statistics, 84 ProcessThread* process_thread, 85 NackSender* nack_sender, 86 // The KeyFrameRequestSender is optional; if not provided, key frame 87 // requests are sent via the internal RtpRtcp module. 88 KeyFrameRequestSender* keyframe_request_sender, 89 video_coding::OnCompleteFrameCallback* complete_frame_callback, 90 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, 91 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 92 ~RtpVideoStreamReceiver2() override; 93 94 void AddReceiveCodec(uint8_t payload_type, 95 const VideoCodec& video_codec, 96 const std::map<std::string, std::string>& codec_params, 97 bool raw_payload); 98 99 void StartReceive(); 100 void StopReceive(); 101 102 // Produces the transport-related timestamps; current_delay_ms is left unset. 103 absl::optional<Syncable::Info> GetSyncInfo() const; 104 105 bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); 106 107 void FrameContinuous(int64_t seq_num); 108 109 void FrameDecoded(int64_t seq_num); 110 111 void SignalNetworkState(NetworkState state); 112 113 // Returns number of different frames seen. GetUniqueFramesSeen()114 int GetUniqueFramesSeen() const { 115 RTC_DCHECK_RUN_ON(&worker_task_checker_); 116 return frame_counter_.GetUniqueSeen(); 117 } 118 119 // Implements RtpPacketSinkInterface. 120 void OnRtpPacket(const RtpPacketReceived& packet) override; 121 122 // Public only for tests. 123 void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, 124 const RtpPacketReceived& rtp_packet, 125 const RTPVideoHeader& video); 126 127 // Implements RecoveredPacketReceiver. 128 void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; 129 130 // Send an RTCP keyframe request. 131 void RequestKeyFrame() override; 132 133 // Implements LossNotificationSender. 134 void SendLossNotification(uint16_t last_decoded_seq_num, 135 uint16_t last_received_seq_num, 136 bool decodability_flag, 137 bool buffering_allowed) override; 138 139 bool IsUlpfecEnabled() const; 140 bool IsRetransmissionsEnabled() const; 141 142 // Returns true if a decryptor is attached and frames can be decrypted. 143 // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame 144 // Decryption not SRTP. 145 bool IsDecryptable() const; 146 147 // Don't use, still experimental. 148 void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers); 149 150 // Implements OnCompleteFrameCallback. 151 void OnCompleteFrame( 152 std::unique_ptr<video_coding::EncodedFrame> frame) override; 153 154 // Implements OnDecryptedFrameCallback. 155 void OnDecryptedFrame( 156 std::unique_ptr<video_coding::RtpFrameObject> frame) override; 157 158 // Implements OnDecryptionStatusChangeCallback. 159 void OnDecryptionStatusChange( 160 FrameDecryptorInterface::Status status) override; 161 162 // Optionally set a frame decryptor after a stream has started. This will not 163 // reset the decoder state. 164 void SetFrameDecryptor( 165 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); 166 167 // Sets a frame transformer after a stream has started, if no transformer 168 // has previously been set. Does not reset the decoder state. 169 void SetDepacketizerToDecoderFrameTransformer( 170 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); 171 172 // Called by VideoReceiveStream when stats are updated. 173 void UpdateRtt(int64_t max_rtt_ms); 174 175 absl::optional<int64_t> LastReceivedPacketMs() const; 176 absl::optional<int64_t> LastReceivedKeyframePacketMs() const; 177 178 // RtpDemuxer only forwards a given RTP packet to one sink. However, some 179 // sinks, such as FlexFEC, might wish to be informed of all of the packets 180 // a given sink receives (or any set of sinks). They may do so by registering 181 // themselves as secondary sinks. 182 void AddSecondarySink(RtpPacketSinkInterface* sink); 183 void RemoveSecondarySink(const RtpPacketSinkInterface* sink); 184 185 // Mozilla modification: VideoReceiveStream2 and friends do not surface RTCP 186 // stats at all, and even on the most recent libwebrtc code there does not 187 // seem to be any support for these stats right now. So, we hack this in. 188 void RemoteRTCPSenderInfo(uint32_t* packet_count, uint32_t* octet_count, 189 int64_t* ntp_timestamp_ms, 190 int64_t* remote_ntp_timestamp_ms) const; 191 192 private: 193 // Implements RtpVideoFrameReceiver. 194 void ManageFrame( 195 std::unique_ptr<video_coding::RtpFrameObject> frame) override; 196 197 // Used for buffering RTCP feedback messages and sending them all together. 198 // Note: 199 // 1. Key frame requests and NACKs are mutually exclusive, with the 200 // former taking precedence over the latter. 201 // 2. Loss notifications are orthogonal to either. (That is, may be sent 202 // alongside either.) 203 class RtcpFeedbackBuffer : public KeyFrameRequestSender, 204 public NackSender, 205 public LossNotificationSender { 206 public: 207 RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, 208 NackSender* nack_sender, 209 LossNotificationSender* loss_notification_sender); 210 211 ~RtcpFeedbackBuffer() override = default; 212 213 // KeyFrameRequestSender implementation. 214 void RequestKeyFrame() override; 215 216 // NackSender implementation. 217 void SendNack(const std::vector<uint16_t>& sequence_numbers, 218 bool buffering_allowed) override; 219 220 // LossNotificationSender implementation. 221 void SendLossNotification(uint16_t last_decoded_seq_num, 222 uint16_t last_received_seq_num, 223 bool decodability_flag, 224 bool buffering_allowed) override; 225 226 // Send all RTCP feedback messages buffered thus far. 227 void SendBufferedRtcpFeedback(); 228 229 private: 230 // LNTF-related state. 231 struct LossNotificationState { LossNotificationStateLossNotificationState232 LossNotificationState(uint16_t last_decoded_seq_num, 233 uint16_t last_received_seq_num, 234 bool decodability_flag) 235 : last_decoded_seq_num(last_decoded_seq_num), 236 last_received_seq_num(last_received_seq_num), 237 decodability_flag(decodability_flag) {} 238 239 uint16_t last_decoded_seq_num; 240 uint16_t last_received_seq_num; 241 bool decodability_flag; 242 }; 243 244 SequenceChecker worker_task_checker_; 245 KeyFrameRequestSender* const key_frame_request_sender_; 246 NackSender* const nack_sender_; 247 LossNotificationSender* const loss_notification_sender_; 248 249 // Key-frame-request-related state. 250 bool request_key_frame_ RTC_GUARDED_BY(worker_task_checker_); 251 252 // NACK-related state. 253 std::vector<uint16_t> nack_sequence_numbers_ 254 RTC_GUARDED_BY(worker_task_checker_); 255 256 absl::optional<LossNotificationState> lntf_state_ 257 RTC_GUARDED_BY(worker_task_checker_); 258 }; 259 enum ParseGenericDependenciesResult { 260 kDropPacket, 261 kHasGenericDescriptor, 262 kNoGenericDescriptor 263 }; 264 265 // Entry point doing non-stats work for a received packet. Called 266 // for the same packet both before and after RED decapsulation. 267 void ReceivePacket(const RtpPacketReceived& packet); 268 // Parses and handles RED headers. 269 // This function assumes that it's being called from only one thread. 270 void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); 271 void NotifyReceiverOfEmptyPacket(uint16_t seq_num); 272 void UpdateHistograms(); 273 bool IsRedEnabled() const; 274 void InsertSpsPpsIntoTracker(uint8_t payload_type); 275 void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); 276 ParseGenericDependenciesResult ParseGenericDependenciesExtension( 277 const RtpPacketReceived& rtp_packet, 278 RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); 279 void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame); 280 281 Clock* const clock_; 282 // Ownership of this object lies with VideoReceiveStream, which owns |this|. 283 const VideoReceiveStream::Config& config_; 284 PacketRouter* const packet_router_; 285 ProcessThread* const process_thread_; 286 287 RemoteNtpTimeEstimator ntp_estimator_; 288 289 RtpHeaderExtensionMap rtp_header_extensions_; 290 // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout 291 // delay that is specified in the received packets. 292 FieldTrialOptional<int> forced_playout_delay_max_ms_; 293 FieldTrialOptional<int> forced_playout_delay_min_ms_; 294 ReceiveStatistics* const rtp_receive_statistics_; 295 std::unique_ptr<UlpfecReceiver> ulpfec_receiver_; 296 297 SequenceChecker worker_task_checker_; 298 bool receiving_ RTC_GUARDED_BY(worker_task_checker_); 299 int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); 300 301 const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; 302 303 video_coding::OnCompleteFrameCallback* complete_frame_callback_; 304 KeyFrameRequestSender* const keyframe_request_sender_; 305 const KeyFrameReqMethod keyframe_request_method_; 306 307 RtcpFeedbackBuffer rtcp_feedback_buffer_; 308 const std::unique_ptr<NackModule2> nack_module_; 309 std::unique_ptr<LossNotificationController> loss_notification_controller_; 310 311 VCMReceiveStatisticsCallback* const vcm_receive_statistics_; 312 video_coding::PacketBuffer packet_buffer_; 313 UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); 314 SeqNumUnwrapper<uint16_t> frame_id_unwrapper_ 315 RTC_GUARDED_BY(worker_task_checker_); 316 317 // Video structure provided in the dependency descriptor in a first packet 318 // of a key frame. It is required to parse dependency descriptor in the 319 // following delta packets. 320 std::unique_ptr<FrameDependencyStructure> video_structure_ 321 RTC_GUARDED_BY(worker_task_checker_); 322 // Frame id of the last frame with the attached video structure. 323 // absl::nullopt when `video_structure_ == nullptr`; 324 absl::optional<int64_t> video_structure_frame_id_ 325 RTC_GUARDED_BY(worker_task_checker_); 326 327 std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_ 328 RTC_GUARDED_BY(worker_task_checker_); 329 absl::optional<VideoCodecType> current_codec_ 330 RTC_GUARDED_BY(worker_task_checker_); 331 uint32_t last_assembled_frame_rtp_timestamp_ 332 RTC_GUARDED_BY(worker_task_checker_); 333 334 std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ 335 RTC_GUARDED_BY(worker_task_checker_); 336 video_coding::H264SpsPpsTracker tracker_ RTC_GUARDED_BY(worker_task_checker_); 337 338 // Maps payload id to the depacketizer. 339 std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_ 340 RTC_GUARDED_BY(worker_task_checker_); 341 342 // TODO(johan): Remove pt_codec_params_ once 343 // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. 344 // Maps a payload type to a map of out-of-band supplied codec parameters. 345 std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_ 346 RTC_GUARDED_BY(worker_task_checker_); 347 int16_t last_payload_type_ RTC_GUARDED_BY(worker_task_checker_) = -1; 348 349 bool has_received_frame_ RTC_GUARDED_BY(worker_task_checker_); 350 351 std::vector<RtpPacketSinkInterface*> secondary_sinks_ 352 RTC_GUARDED_BY(worker_task_checker_); 353 354 absl::optional<uint32_t> last_received_rtp_timestamp_ 355 RTC_GUARDED_BY(worker_task_checker_); 356 absl::optional<int64_t> last_received_rtp_system_time_ms_ 357 RTC_GUARDED_BY(worker_task_checker_); 358 359 // Handles incoming encrypted frames and forwards them to the 360 // rtp_reference_finder if they are decryptable. 361 std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_ 362 RTC_PT_GUARDED_BY(worker_task_checker_); 363 bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_); 364 absl::optional<ColorSpace> last_color_space_; 365 366 AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ 367 RTC_GUARDED_BY(worker_task_checker_); 368 369 int64_t last_completed_picture_id_ = 0; 370 371 rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> 372 frame_transformer_delegate_; 373 }; 374 375 } // namespace webrtc 376 377 #endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ 378