1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
12 #define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
13 
14 #include <atomic>
15 #include <list>
16 #include <map>
17 #include <memory>
18 #include <string>
19 #include <vector>
20 
21 #include "absl/types/optional.h"
22 #include "api/array_view.h"
23 #include "api/crypto/frame_decryptor_interface.h"
24 #include "api/video/color_space.h"
25 #include "api/video_codecs/video_codec.h"
26 #include "call/rtp_packet_sink_interface.h"
27 #include "call/syncable.h"
28 #include "call/video_receive_stream.h"
29 #include "modules/rtp_rtcp/include/receive_statistics.h"
30 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
31 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
32 #include "modules/rtp_rtcp/include/rtp_rtcp.h"
33 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
34 #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
35 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
36 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
37 #include "modules/rtp_rtcp/source/rtp_video_header.h"
38 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
39 #include "modules/video_coding/h264_sps_pps_tracker.h"
40 #include "modules/video_coding/loss_notification_controller.h"
41 #include "modules/video_coding/packet_buffer.h"
42 #include "modules/video_coding/rtp_frame_reference_finder.h"
43 #include "modules/video_coding/unique_timestamp_counter.h"
44 #include "rtc_base/constructor_magic.h"
45 #include "rtc_base/experiments/field_trial_parser.h"
46 #include "rtc_base/numerics/sequence_number_util.h"
47 #include "rtc_base/synchronization/mutex.h"
48 #include "rtc_base/synchronization/sequence_checker.h"
49 #include "rtc_base/thread_annotations.h"
50 #include "rtc_base/thread_checker.h"
51 #include "video/buffered_frame_decryptor.h"
52 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
53 
54 namespace webrtc {
55 
56 class DEPRECATED_NackModule;
57 class PacketRouter;
58 class ProcessThread;
59 class ReceiveStatistics;
60 class ReceiveStatisticsProxy;
61 class RtcpRttStats;
62 class RtpPacketReceived;
63 class Transport;
64 class UlpfecReceiver;
65 
66 class RtpVideoStreamReceiver : public LossNotificationSender,
67                                public RecoveredPacketReceiver,
68                                public RtpPacketSinkInterface,
69                                public KeyFrameRequestSender,
70                                public video_coding::OnCompleteFrameCallback,
71                                public OnDecryptedFrameCallback,
72                                public OnDecryptionStatusChangeCallback,
73                                public RtpVideoFrameReceiver {
74  public:
75   // DEPRECATED due to dependency on ReceiveStatisticsProxy.
76   RtpVideoStreamReceiver(
77       Clock* clock,
78       Transport* transport,
79       RtcpRttStats* rtt_stats,
80       // The packet router is optional; if provided, the RtpRtcp module for this
81       // stream is registered as a candidate for sending REMB and transport
82       // feedback.
83       PacketRouter* packet_router,
84       const VideoReceiveStream::Config* config,
85       ReceiveStatistics* rtp_receive_statistics,
86       ReceiveStatisticsProxy* receive_stats_proxy,
87       ProcessThread* process_thread,
88       NackSender* nack_sender,
89       // The KeyFrameRequestSender is optional; if not provided, key frame
90       // requests are sent via the internal RtpRtcp module.
91       KeyFrameRequestSender* keyframe_request_sender,
92       video_coding::OnCompleteFrameCallback* complete_frame_callback,
93       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
94       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
95 
96   RtpVideoStreamReceiver(
97       Clock* clock,
98       Transport* transport,
99       RtcpRttStats* rtt_stats,
100       // The packet router is optional; if provided, the RtpRtcp module for this
101       // stream is registered as a candidate for sending REMB and transport
102       // feedback.
103       PacketRouter* packet_router,
104       const VideoReceiveStream::Config* config,
105       ReceiveStatistics* rtp_receive_statistics,
106       RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
107       RtcpCnameCallback* rtcp_cname_callback,
108       ProcessThread* process_thread,
109       NackSender* nack_sender,
110       // The KeyFrameRequestSender is optional; if not provided, key frame
111       // requests are sent via the internal RtpRtcp module.
112       KeyFrameRequestSender* keyframe_request_sender,
113       video_coding::OnCompleteFrameCallback* complete_frame_callback,
114       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
115       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
116   ~RtpVideoStreamReceiver() override;
117 
118   void AddReceiveCodec(uint8_t payload_type,
119                        const VideoCodec& video_codec,
120                        const std::map<std::string, std::string>& codec_params,
121                        bool raw_payload);
122 
rtp_rtcp()123   RtpRtcp* rtp_rtcp() const { return rtp_rtcp_.get(); }
124 
125   void StartReceive();
126   void StopReceive();
127 
128   // Produces the transport-related timestamps; current_delay_ms is left unset.
129   absl::optional<Syncable::Info> GetSyncInfo() const;
130 
131   bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length);
132 
133   void FrameContinuous(int64_t seq_num);
134 
135   void FrameDecoded(int64_t seq_num);
136 
137   void SignalNetworkState(NetworkState state);
138 
139   // Returns number of different frames seen.
GetUniqueFramesSeen()140   int GetUniqueFramesSeen() const {
141     RTC_DCHECK_RUN_ON(&worker_task_checker_);
142     return frame_counter_.GetUniqueSeen();
143   }
144 
145   // Implements RtpPacketSinkInterface.
146   void OnRtpPacket(const RtpPacketReceived& packet) override;
147 
148   // Public only for tests.
149   void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,
150                              const RtpPacketReceived& rtp_packet,
151                              const RTPVideoHeader& video);
152 
153   // Implements RecoveredPacketReceiver.
154   void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override;
155 
156   // Send an RTCP keyframe request.
157   void RequestKeyFrame() override;
158 
159   // Implements LossNotificationSender.
160   void SendLossNotification(uint16_t last_decoded_seq_num,
161                             uint16_t last_received_seq_num,
162                             bool decodability_flag,
163                             bool buffering_allowed) override;
164 
165   bool IsUlpfecEnabled() const;
166   bool IsRetransmissionsEnabled() const;
167 
168   // Returns true if a decryptor is attached and frames can be decrypted.
169   // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame
170   // Decryption not SRTP.
171   bool IsDecryptable() const;
172 
173   // Don't use, still experimental.
174   void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers);
175 
176   // Implements OnCompleteFrameCallback.
177   void OnCompleteFrame(
178       std::unique_ptr<video_coding::EncodedFrame> frame) override;
179 
180   // Implements OnDecryptedFrameCallback.
181   void OnDecryptedFrame(
182       std::unique_ptr<video_coding::RtpFrameObject> frame) override;
183 
184   // Implements OnDecryptionStatusChangeCallback.
185   void OnDecryptionStatusChange(
186       FrameDecryptorInterface::Status status) override;
187 
188   // Optionally set a frame decryptor after a stream has started. This will not
189   // reset the decoder state.
190   void SetFrameDecryptor(
191       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor);
192 
193   // Sets a frame transformer after a stream has started, if no transformer
194   // has previously been set. Does not reset the decoder state.
195   void SetDepacketizerToDecoderFrameTransformer(
196       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
197 
198   // Called by VideoReceiveStream when stats are updated.
199   void UpdateRtt(int64_t max_rtt_ms);
200 
201   absl::optional<int64_t> LastReceivedPacketMs() const;
202   absl::optional<int64_t> LastReceivedKeyframePacketMs() const;
203 
204   // RtpDemuxer only forwards a given RTP packet to one sink. However, some
205   // sinks, such as FlexFEC, might wish to be informed of all of the packets
206   // a given sink receives (or any set of sinks). They may do so by registering
207   // themselves as secondary sinks.
208   void AddSecondarySink(RtpPacketSinkInterface* sink);
209   void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
210 
211  private:
212   // Implements RtpVideoFrameReceiver.
213   void ManageFrame(
214       std::unique_ptr<video_coding::RtpFrameObject> frame) override;
215 
216   // Used for buffering RTCP feedback messages and sending them all together.
217   // Note:
218   // 1. Key frame requests and NACKs are mutually exclusive, with the
219   //    former taking precedence over the latter.
220   // 2. Loss notifications are orthogonal to either. (That is, may be sent
221   //    alongside either.)
222   class RtcpFeedbackBuffer : public KeyFrameRequestSender,
223                              public NackSender,
224                              public LossNotificationSender {
225    public:
226     RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender,
227                        NackSender* nack_sender,
228                        LossNotificationSender* loss_notification_sender);
229 
230     ~RtcpFeedbackBuffer() override = default;
231 
232     // KeyFrameRequestSender implementation.
233     void RequestKeyFrame() RTC_LOCKS_EXCLUDED(mutex_) override;
234 
235     // NackSender implementation.
236     void SendNack(const std::vector<uint16_t>& sequence_numbers,
237                   bool buffering_allowed) RTC_LOCKS_EXCLUDED(mutex_) override;
238 
239     // LossNotificationSender implementation.
240     void SendLossNotification(uint16_t last_decoded_seq_num,
241                               uint16_t last_received_seq_num,
242                               bool decodability_flag,
243                               bool buffering_allowed)
244         RTC_LOCKS_EXCLUDED(mutex_) override;
245 
246     // Send all RTCP feedback messages buffered thus far.
247     void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_);
248 
249    private:
250     // LNTF-related state.
251     struct LossNotificationState {
LossNotificationStateLossNotificationState252       LossNotificationState(uint16_t last_decoded_seq_num,
253                             uint16_t last_received_seq_num,
254                             bool decodability_flag)
255           : last_decoded_seq_num(last_decoded_seq_num),
256             last_received_seq_num(last_received_seq_num),
257             decodability_flag(decodability_flag) {}
258 
259       uint16_t last_decoded_seq_num;
260       uint16_t last_received_seq_num;
261       bool decodability_flag;
262     };
263     struct ConsumedRtcpFeedback {
264       bool request_key_frame = false;
265       std::vector<uint16_t> nack_sequence_numbers;
266       absl::optional<LossNotificationState> lntf_state;
267     };
268 
269     ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(mutex_);
270     ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked()
271         RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
272     // This method is called both with and without mutex_ held.
273     void SendRtcpFeedback(ConsumedRtcpFeedback feedback);
274 
275     KeyFrameRequestSender* const key_frame_request_sender_;
276     NackSender* const nack_sender_;
277     LossNotificationSender* const loss_notification_sender_;
278 
279     // NACKs are accessible from two threads due to nack_module_ being a module.
280     Mutex mutex_;
281 
282     // Key-frame-request-related state.
283     bool request_key_frame_ RTC_GUARDED_BY(mutex_);
284 
285     // NACK-related state.
286     std::vector<uint16_t> nack_sequence_numbers_ RTC_GUARDED_BY(mutex_);
287 
288     absl::optional<LossNotificationState> lntf_state_ RTC_GUARDED_BY(mutex_);
289   };
290   enum ParseGenericDependenciesResult {
291     kDropPacket,
292     kHasGenericDescriptor,
293     kNoGenericDescriptor
294   };
295 
296   // Entry point doing non-stats work for a received packet. Called
297   // for the same packet both before and after RED decapsulation.
298   void ReceivePacket(const RtpPacketReceived& packet);
299   // Parses and handles RED headers.
300   // This function assumes that it's being called from only one thread.
301   void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet);
302   void NotifyReceiverOfEmptyPacket(uint16_t seq_num);
303   void UpdateHistograms();
304   bool IsRedEnabled() const;
305   void InsertSpsPpsIntoTracker(uint8_t payload_type);
306   void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result);
307   ParseGenericDependenciesResult ParseGenericDependenciesExtension(
308       const RtpPacketReceived& rtp_packet,
309       RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_);
310   void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
311 
312   Clock* const clock_;
313   // Ownership of this object lies with VideoReceiveStream, which owns |this|.
314   const VideoReceiveStream::Config& config_;
315   PacketRouter* const packet_router_;
316   ProcessThread* const process_thread_;
317 
318   RemoteNtpTimeEstimator ntp_estimator_;
319 
320   RtpHeaderExtensionMap rtp_header_extensions_;
321   // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout
322   // delay that is specified in the received packets.
323   FieldTrialOptional<int> forced_playout_delay_max_ms_;
324   FieldTrialOptional<int> forced_playout_delay_min_ms_;
325   ReceiveStatistics* const rtp_receive_statistics_;
326   std::unique_ptr<UlpfecReceiver> ulpfec_receiver_;
327 
328   SequenceChecker worker_task_checker_;
329   bool receiving_ RTC_GUARDED_BY(worker_task_checker_);
330   int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_);
331 
332   const std::unique_ptr<RtpRtcp> rtp_rtcp_;
333 
334   video_coding::OnCompleteFrameCallback* complete_frame_callback_;
335   KeyFrameRequestSender* const keyframe_request_sender_;
336 
337   RtcpFeedbackBuffer rtcp_feedback_buffer_;
338   std::unique_ptr<DEPRECATED_NackModule> nack_module_;
339   std::unique_ptr<LossNotificationController> loss_notification_controller_;
340 
341   video_coding::PacketBuffer packet_buffer_;
342   UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_);
343   SeqNumUnwrapper<uint16_t> frame_id_unwrapper_
344       RTC_GUARDED_BY(worker_task_checker_);
345 
346   // Video structure provided in the dependency descriptor in a first packet
347   // of a key frame. It is required to parse dependency descriptor in the
348   // following delta packets.
349   std::unique_ptr<FrameDependencyStructure> video_structure_
350       RTC_GUARDED_BY(worker_task_checker_);
351   // Frame id of the last frame with the attached video structure.
352   // absl::nullopt when `video_structure_ == nullptr`;
353   absl::optional<int64_t> video_structure_frame_id_
354       RTC_GUARDED_BY(worker_task_checker_);
355 
356   Mutex reference_finder_lock_;
357   std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_
358       RTC_GUARDED_BY(reference_finder_lock_);
359   absl::optional<VideoCodecType> current_codec_;
360   uint32_t last_assembled_frame_rtp_timestamp_;
361 
362   Mutex last_seq_num_mutex_;
363   std::map<int64_t, uint16_t> last_seq_num_for_pic_id_
364       RTC_GUARDED_BY(last_seq_num_mutex_);
365   video_coding::H264SpsPpsTracker tracker_;
366 
367   // Maps payload id to the depacketizer.
368   std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_;
369 
370   // TODO(johan): Remove pt_codec_params_ once
371   // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved.
372   // Maps a payload type to a map of out-of-band supplied codec parameters.
373   std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_;
374   int16_t last_payload_type_ = -1;
375 
376   bool has_received_frame_;
377 
378   std::vector<RtpPacketSinkInterface*> secondary_sinks_
379       RTC_GUARDED_BY(worker_task_checker_);
380 
381   // Info for GetSyncInfo is updated on network or worker thread, and queried on
382   // the worker thread.
383   mutable Mutex sync_info_lock_;
384   absl::optional<uint32_t> last_received_rtp_timestamp_
385       RTC_GUARDED_BY(sync_info_lock_);
386   absl::optional<int64_t> last_received_rtp_system_time_ms_
387       RTC_GUARDED_BY(sync_info_lock_);
388 
389   // Used to validate the buffered frame decryptor is always run on the correct
390   // thread.
391   rtc::ThreadChecker network_tc_;
392   // Handles incoming encrypted frames and forwards them to the
393   // rtp_reference_finder if they are decryptable.
394   std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_
395       RTC_PT_GUARDED_BY(network_tc_);
396   std::atomic<bool> frames_decryptable_;
397   absl::optional<ColorSpace> last_color_space_;
398 
399   AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_
400       RTC_GUARDED_BY(worker_task_checker_);
401 
402   int64_t last_completed_picture_id_ = 0;
403 
404   rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
405       frame_transformer_delegate_;
406 };
407 
408 }  // namespace webrtc
409 
410 #endif  // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_
411