1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
12 #define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
13 
14 #include <map>
15 #include <memory>
16 #include <string>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/crypto/frame_decryptor_interface.h"
21 #include "api/video/color_space.h"
22 #include "api/video_codecs/video_codec.h"
23 #include "call/rtp_packet_sink_interface.h"
24 #include "call/syncable.h"
25 #include "call/video_receive_stream.h"
26 #include "modules/rtp_rtcp/include/receive_statistics.h"
27 #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
28 #include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
29 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
30 #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
31 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
32 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
33 #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
34 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
35 #include "modules/rtp_rtcp/source/rtp_video_header.h"
36 #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
37 #include "modules/video_coding/h264_sps_pps_tracker.h"
38 #include "modules/video_coding/loss_notification_controller.h"
39 #include "modules/video_coding/packet_buffer.h"
40 #include "modules/video_coding/rtp_frame_reference_finder.h"
41 #include "modules/video_coding/unique_timestamp_counter.h"
42 #include "rtc_base/constructor_magic.h"
43 #include "rtc_base/experiments/field_trial_parser.h"
44 #include "rtc_base/numerics/sequence_number_util.h"
45 #include "rtc_base/synchronization/sequence_checker.h"
46 #include "rtc_base/thread_annotations.h"
47 #include "video/buffered_frame_decryptor.h"
48 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
49 
50 namespace webrtc {
51 
52 class NackModule2;
53 class PacketRouter;
54 class ProcessThread;
55 class ReceiveStatistics;
56 class RtcpRttStats;
57 class RtpPacketReceived;
58 class Transport;
59 class UlpfecReceiver;
60 
61 class RtpVideoStreamReceiver2 : public LossNotificationSender,
62                                 public RecoveredPacketReceiver,
63                                 public RtpPacketSinkInterface,
64                                 public KeyFrameRequestSender,
65                                 public video_coding::OnCompleteFrameCallback,
66                                 public OnDecryptedFrameCallback,
67                                 public OnDecryptionStatusChangeCallback,
68                                 public RtpVideoFrameReceiver {
69  public:
70   RtpVideoStreamReceiver2(
71       TaskQueueBase* current_queue,
72       Clock* clock,
73       Transport* transport,
74       RtcpRttStats* rtt_stats,
75       // The packet router is optional; if provided, the RtpRtcp module for this
76       // stream is registered as a candidate for sending REMB and transport
77       // feedback.
78       PacketRouter* packet_router,
79       const VideoReceiveStream::Config* config,
80       ReceiveStatistics* rtp_receive_statistics,
81       RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
82       RtcpCnameCallback* rtcp_cname_callback,
83       VCMReceiveStatisticsCallback* vcm_receive_statistics,
84       ProcessThread* process_thread,
85       NackSender* nack_sender,
86       // The KeyFrameRequestSender is optional; if not provided, key frame
87       // requests are sent via the internal RtpRtcp module.
88       KeyFrameRequestSender* keyframe_request_sender,
89       video_coding::OnCompleteFrameCallback* complete_frame_callback,
90       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
91       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
92   ~RtpVideoStreamReceiver2() override;
93 
94   void AddReceiveCodec(uint8_t payload_type,
95                        const VideoCodec& video_codec,
96                        const std::map<std::string, std::string>& codec_params,
97                        bool raw_payload);
98 
99   void StartReceive();
100   void StopReceive();
101 
102   // Produces the transport-related timestamps; current_delay_ms is left unset.
103   absl::optional<Syncable::Info> GetSyncInfo() const;
104 
105   bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length);
106 
107   void FrameContinuous(int64_t seq_num);
108 
109   void FrameDecoded(int64_t seq_num);
110 
111   void SignalNetworkState(NetworkState state);
112 
113   // Returns number of different frames seen.
GetUniqueFramesSeen()114   int GetUniqueFramesSeen() const {
115     RTC_DCHECK_RUN_ON(&worker_task_checker_);
116     return frame_counter_.GetUniqueSeen();
117   }
118 
119   // Implements RtpPacketSinkInterface.
120   void OnRtpPacket(const RtpPacketReceived& packet) override;
121 
122   // Public only for tests.
123   void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,
124                              const RtpPacketReceived& rtp_packet,
125                              const RTPVideoHeader& video);
126 
127   // Implements RecoveredPacketReceiver.
128   void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override;
129 
130   // Send an RTCP keyframe request.
131   void RequestKeyFrame() override;
132 
133   // Implements LossNotificationSender.
134   void SendLossNotification(uint16_t last_decoded_seq_num,
135                             uint16_t last_received_seq_num,
136                             bool decodability_flag,
137                             bool buffering_allowed) override;
138 
139   bool IsUlpfecEnabled() const;
140   bool IsRetransmissionsEnabled() const;
141 
142   // Returns true if a decryptor is attached and frames can be decrypted.
143   // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame
144   // Decryption not SRTP.
145   bool IsDecryptable() const;
146 
147   // Don't use, still experimental.
148   void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers);
149 
150   // Implements OnCompleteFrameCallback.
151   void OnCompleteFrame(
152       std::unique_ptr<video_coding::EncodedFrame> frame) override;
153 
154   // Implements OnDecryptedFrameCallback.
155   void OnDecryptedFrame(
156       std::unique_ptr<video_coding::RtpFrameObject> frame) override;
157 
158   // Implements OnDecryptionStatusChangeCallback.
159   void OnDecryptionStatusChange(
160       FrameDecryptorInterface::Status status) override;
161 
162   // Optionally set a frame decryptor after a stream has started. This will not
163   // reset the decoder state.
164   void SetFrameDecryptor(
165       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor);
166 
167   // Sets a frame transformer after a stream has started, if no transformer
168   // has previously been set. Does not reset the decoder state.
169   void SetDepacketizerToDecoderFrameTransformer(
170       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
171 
172   // Called by VideoReceiveStream when stats are updated.
173   void UpdateRtt(int64_t max_rtt_ms);
174 
175   absl::optional<int64_t> LastReceivedPacketMs() const;
176   absl::optional<int64_t> LastReceivedKeyframePacketMs() const;
177 
178   // RtpDemuxer only forwards a given RTP packet to one sink. However, some
179   // sinks, such as FlexFEC, might wish to be informed of all of the packets
180   // a given sink receives (or any set of sinks). They may do so by registering
181   // themselves as secondary sinks.
182   void AddSecondarySink(RtpPacketSinkInterface* sink);
183   void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
184 
185   // Mozilla modification: VideoReceiveStream2 and friends do not surface RTCP
186   // stats at all, and even on the most recent libwebrtc code there does not
187   // seem to be any support for these stats right now. So, we hack this in.
188   void RemoteRTCPSenderInfo(uint32_t* packet_count, uint32_t* octet_count,
189                             int64_t* ntp_timestamp_ms,
190                             int64_t* remote_ntp_timestamp_ms) const;
191 
192  private:
193   // Implements RtpVideoFrameReceiver.
194   void ManageFrame(
195       std::unique_ptr<video_coding::RtpFrameObject> frame) override;
196 
197   // Used for buffering RTCP feedback messages and sending them all together.
198   // Note:
199   // 1. Key frame requests and NACKs are mutually exclusive, with the
200   //    former taking precedence over the latter.
201   // 2. Loss notifications are orthogonal to either. (That is, may be sent
202   //    alongside either.)
203   class RtcpFeedbackBuffer : public KeyFrameRequestSender,
204                              public NackSender,
205                              public LossNotificationSender {
206    public:
207     RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender,
208                        NackSender* nack_sender,
209                        LossNotificationSender* loss_notification_sender);
210 
211     ~RtcpFeedbackBuffer() override = default;
212 
213     // KeyFrameRequestSender implementation.
214     void RequestKeyFrame() override;
215 
216     // NackSender implementation.
217     void SendNack(const std::vector<uint16_t>& sequence_numbers,
218                   bool buffering_allowed) override;
219 
220     // LossNotificationSender implementation.
221     void SendLossNotification(uint16_t last_decoded_seq_num,
222                               uint16_t last_received_seq_num,
223                               bool decodability_flag,
224                               bool buffering_allowed) override;
225 
226     // Send all RTCP feedback messages buffered thus far.
227     void SendBufferedRtcpFeedback();
228 
229    private:
230     // LNTF-related state.
231     struct LossNotificationState {
LossNotificationStateLossNotificationState232       LossNotificationState(uint16_t last_decoded_seq_num,
233                             uint16_t last_received_seq_num,
234                             bool decodability_flag)
235           : last_decoded_seq_num(last_decoded_seq_num),
236             last_received_seq_num(last_received_seq_num),
237             decodability_flag(decodability_flag) {}
238 
239       uint16_t last_decoded_seq_num;
240       uint16_t last_received_seq_num;
241       bool decodability_flag;
242     };
243 
244     SequenceChecker worker_task_checker_;
245     KeyFrameRequestSender* const key_frame_request_sender_;
246     NackSender* const nack_sender_;
247     LossNotificationSender* const loss_notification_sender_;
248 
249     // Key-frame-request-related state.
250     bool request_key_frame_ RTC_GUARDED_BY(worker_task_checker_);
251 
252     // NACK-related state.
253     std::vector<uint16_t> nack_sequence_numbers_
254         RTC_GUARDED_BY(worker_task_checker_);
255 
256     absl::optional<LossNotificationState> lntf_state_
257         RTC_GUARDED_BY(worker_task_checker_);
258   };
259   enum ParseGenericDependenciesResult {
260     kDropPacket,
261     kHasGenericDescriptor,
262     kNoGenericDescriptor
263   };
264 
265   // Entry point doing non-stats work for a received packet. Called
266   // for the same packet both before and after RED decapsulation.
267   void ReceivePacket(const RtpPacketReceived& packet);
268   // Parses and handles RED headers.
269   // This function assumes that it's being called from only one thread.
270   void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet);
271   void NotifyReceiverOfEmptyPacket(uint16_t seq_num);
272   void UpdateHistograms();
273   bool IsRedEnabled() const;
274   void InsertSpsPpsIntoTracker(uint8_t payload_type);
275   void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result);
276   ParseGenericDependenciesResult ParseGenericDependenciesExtension(
277       const RtpPacketReceived& rtp_packet,
278       RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_);
279   void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
280 
281   Clock* const clock_;
282   // Ownership of this object lies with VideoReceiveStream, which owns |this|.
283   const VideoReceiveStream::Config& config_;
284   PacketRouter* const packet_router_;
285   ProcessThread* const process_thread_;
286 
287   RemoteNtpTimeEstimator ntp_estimator_;
288 
289   RtpHeaderExtensionMap rtp_header_extensions_;
290   // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout
291   // delay that is specified in the received packets.
292   FieldTrialOptional<int> forced_playout_delay_max_ms_;
293   FieldTrialOptional<int> forced_playout_delay_min_ms_;
294   ReceiveStatistics* const rtp_receive_statistics_;
295   std::unique_ptr<UlpfecReceiver> ulpfec_receiver_;
296 
297   SequenceChecker worker_task_checker_;
298   bool receiving_ RTC_GUARDED_BY(worker_task_checker_);
299   int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_);
300 
301   const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
302 
303   video_coding::OnCompleteFrameCallback* complete_frame_callback_;
304   KeyFrameRequestSender* const keyframe_request_sender_;
305   const KeyFrameReqMethod keyframe_request_method_;
306 
307   RtcpFeedbackBuffer rtcp_feedback_buffer_;
308   const std::unique_ptr<NackModule2> nack_module_;
309   std::unique_ptr<LossNotificationController> loss_notification_controller_;
310 
311   VCMReceiveStatisticsCallback* const vcm_receive_statistics_;
312   video_coding::PacketBuffer packet_buffer_;
313   UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_);
314   SeqNumUnwrapper<uint16_t> frame_id_unwrapper_
315       RTC_GUARDED_BY(worker_task_checker_);
316 
317   // Video structure provided in the dependency descriptor in a first packet
318   // of a key frame. It is required to parse dependency descriptor in the
319   // following delta packets.
320   std::unique_ptr<FrameDependencyStructure> video_structure_
321       RTC_GUARDED_BY(worker_task_checker_);
322   // Frame id of the last frame with the attached video structure.
323   // absl::nullopt when `video_structure_ == nullptr`;
324   absl::optional<int64_t> video_structure_frame_id_
325       RTC_GUARDED_BY(worker_task_checker_);
326 
327   std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_
328       RTC_GUARDED_BY(worker_task_checker_);
329   absl::optional<VideoCodecType> current_codec_
330       RTC_GUARDED_BY(worker_task_checker_);
331   uint32_t last_assembled_frame_rtp_timestamp_
332       RTC_GUARDED_BY(worker_task_checker_);
333 
334   std::map<int64_t, uint16_t> last_seq_num_for_pic_id_
335       RTC_GUARDED_BY(worker_task_checker_);
336   video_coding::H264SpsPpsTracker tracker_ RTC_GUARDED_BY(worker_task_checker_);
337 
338   // Maps payload id to the depacketizer.
339   std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_
340       RTC_GUARDED_BY(worker_task_checker_);
341 
342   // TODO(johan): Remove pt_codec_params_ once
343   // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved.
344   // Maps a payload type to a map of out-of-band supplied codec parameters.
345   std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_
346       RTC_GUARDED_BY(worker_task_checker_);
347   int16_t last_payload_type_ RTC_GUARDED_BY(worker_task_checker_) = -1;
348 
349   bool has_received_frame_ RTC_GUARDED_BY(worker_task_checker_);
350 
351   std::vector<RtpPacketSinkInterface*> secondary_sinks_
352       RTC_GUARDED_BY(worker_task_checker_);
353 
354   absl::optional<uint32_t> last_received_rtp_timestamp_
355       RTC_GUARDED_BY(worker_task_checker_);
356   absl::optional<int64_t> last_received_rtp_system_time_ms_
357       RTC_GUARDED_BY(worker_task_checker_);
358 
359   // Handles incoming encrypted frames and forwards them to the
360   // rtp_reference_finder if they are decryptable.
361   std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_
362       RTC_PT_GUARDED_BY(worker_task_checker_);
363   bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_);
364   absl::optional<ColorSpace> last_color_space_;
365 
366   AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_
367       RTC_GUARDED_BY(worker_task_checker_);
368 
369   int64_t last_completed_picture_id_ = 0;
370 
371   rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
372       frame_transformer_delegate_;
373 };
374 
375 }  // namespace webrtc
376 
377 #endif  // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
378