1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM2_H_
12 #define VIDEO_VIDEO_RECEIVE_STREAM2_H_
13 
14 #include <memory>
15 #include <vector>
16 
17 #include "api/task_queue/task_queue_factory.h"
18 #include "api/units/timestamp.h"
19 #include "api/video/recordable_encoded_frame.h"
20 #include "call/rtp_packet_sink_interface.h"
21 #include "call/syncable.h"
22 #include "call/video_receive_stream.h"
23 #include "modules/rtp_rtcp/include/flexfec_receiver.h"
24 #include "modules/rtp_rtcp/source/source_tracker.h"
25 #include "modules/video_coding/frame_buffer2.h"
26 #include "modules/video_coding/video_receiver2.h"
27 #include "rtc_base/synchronization/sequence_checker.h"
28 #include "rtc_base/task_queue.h"
29 #include "rtc_base/task_utils/pending_task_safety_flag.h"
30 #include "system_wrappers/include/clock.h"
31 #include "video/receive_statistics_proxy2.h"
32 #include "video/rtp_streams_synchronizer2.h"
33 #include "video/rtp_video_stream_receiver2.h"
34 #include "video/transport_adapter.h"
35 #include "video/video_stream_decoder2.h"
36 
37 namespace webrtc {
38 
39 class ProcessThread;
40 class RtpStreamReceiverInterface;
41 class RtpStreamReceiverControllerInterface;
42 class RtxReceiveStream;
43 class VCMTiming;
44 
45 namespace internal {
46 
47 class CallStats;
48 
49 // Utility struct for grabbing metadata from a VideoFrame and processing it
50 // asynchronously without needing the actual frame data.
51 // Additionally the caller can bundle information from the current clock
52 // when the metadata is captured, for accurate reporting and not needeing
53 // multiple calls to clock->Now().
54 struct VideoFrameMetaData {
VideoFrameMetaDataVideoFrameMetaData55   VideoFrameMetaData(const webrtc::VideoFrame& frame, Timestamp now)
56       : rtp_timestamp(frame.timestamp()),
57         timestamp_us(frame.timestamp_us()),
58         ntp_time_ms(frame.ntp_time_ms()),
59         width(frame.width()),
60         height(frame.height()),
61         decode_timestamp(now) {}
62 
render_time_msVideoFrameMetaData63   int64_t render_time_ms() const {
64     return timestamp_us / rtc::kNumMicrosecsPerMillisec;
65   }
66 
67   const uint32_t rtp_timestamp;
68   const int64_t timestamp_us;
69   const int64_t ntp_time_ms;
70   const int width;
71   const int height;
72 
73   const Timestamp decode_timestamp;
74 };
75 
76 class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
77                             public rtc::VideoSinkInterface<VideoFrame>,
78                             public NackSender,
79                             public video_coding::OnCompleteFrameCallback,
80                             public Syncable,
81                             public CallStatsObserver {
82  public:
83   // The default number of milliseconds to pass before re-requesting a key frame
84   // to be sent.
85   static constexpr int kMaxWaitForKeyFrameMs = 200;
86 
87   VideoReceiveStream2(TaskQueueFactory* task_queue_factory,
88                       TaskQueueBase* current_queue,
89                       RtpStreamReceiverControllerInterface* receiver_controller,
90                       int num_cpu_cores,
91                       PacketRouter* packet_router,
92                       VideoReceiveStream::Config config,
93                       ProcessThread* process_thread,
94                       CallStats* call_stats,
95                       Clock* clock,
96                       VCMTiming* timing);
97   ~VideoReceiveStream2() override;
98 
config()99   const Config& config() const { return config_; }
100 
101   void SignalNetworkState(NetworkState state);
102   bool DeliverRtcp(const uint8_t* packet, size_t length);
103 
104   void SetSync(Syncable* audio_syncable);
105 
106   // Implements webrtc::VideoReceiveStream.
107   void Start() override;
108   void Stop() override;
109 
110   webrtc::VideoReceiveStream::Stats GetStats() const override;
111 
112   void AddSecondarySink(RtpPacketSinkInterface* sink) override;
113   void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override;
114 
115   // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called
116   // from webrtc/api level and requested by user code. For e.g. blink/js layer
117   // in Chromium.
118   bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
119   int GetBaseMinimumPlayoutDelayMs() const override;
120 
121   void SetFrameDecryptor(
122       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override;
123   void SetDepacketizerToDecoderFrameTransformer(
124       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override;
125 
126   // Implements rtc::VideoSinkInterface<VideoFrame>.
127   void OnFrame(const VideoFrame& video_frame) override;
128 
129   // Implements NackSender.
130   // For this particular override of the interface,
131   // only (buffering_allowed == true) is acceptable.
132   void SendNack(const std::vector<uint16_t>& sequence_numbers,
133                 bool buffering_allowed) override;
134 
135   // Implements video_coding::OnCompleteFrameCallback.
136   void OnCompleteFrame(
137       std::unique_ptr<video_coding::EncodedFrame> frame) override;
138 
139   // Implements CallStatsObserver::OnRttUpdate
140   void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override;
141 
142   // Implements Syncable.
143   uint32_t id() const override;
144   absl::optional<Syncable::Info> GetInfo() const override;
145   bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
146                               int64_t* time_ms) const override;
147   void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,
148                                          int64_t time_ms) override;
149 
150   // SetMinimumPlayoutDelay is only called by A/V sync.
151   bool SetMinimumPlayoutDelay(int delay_ms) override;
152 
153   std::vector<webrtc::RtpSource> GetSources() const override;
154 
155   RecordingState SetAndGetRecordingState(RecordingState state,
156                                          bool generate_key_frame) override;
157   void GenerateKeyFrame() override;
158 
159  private:
160   int64_t GetMaxWaitMs() const RTC_RUN_ON(decode_queue_);
161   void StartNextDecode() RTC_RUN_ON(decode_queue_);
162   void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame)
163       RTC_RUN_ON(decode_queue_);
164   void HandleFrameBufferTimeout(int64_t now_ms, int64_t wait_ms)
165       RTC_RUN_ON(worker_sequence_checker_);
166   void UpdatePlayoutDelays() const
167       RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_sequence_checker_);
168   void RequestKeyFrame(int64_t timestamp_ms)
169       RTC_RUN_ON(worker_sequence_checker_);
170   void HandleKeyFrameGeneration(bool received_frame_is_keyframe,
171                                 int64_t now_ms,
172                                 bool always_request_key_frame,
173                                 bool keyframe_request_is_due)
174       RTC_RUN_ON(worker_sequence_checker_);
175   bool IsReceivingKeyFrame(int64_t timestamp_ms) const
176       RTC_RUN_ON(worker_sequence_checker_);
177 
178   void UpdateHistograms();
179 
180   SequenceChecker worker_sequence_checker_;
181   SequenceChecker module_process_sequence_checker_;
182 
183   TaskQueueFactory* const task_queue_factory_;
184 
185   TransportAdapter transport_adapter_;
186   const VideoReceiveStream::Config config_;
187   const int num_cpu_cores_;
188   TaskQueueBase* const worker_thread_;
189   Clock* const clock_;
190 
191   CallStats* const call_stats_;
192 
193   bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false;
194   bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true;
195 
196   SourceTracker source_tracker_;
197   ReceiveStatisticsProxy stats_proxy_;
198   // Shared by media and rtx stream receivers, since the latter has no RtpRtcp
199   // module of its own.
200   const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
201 
202   std::unique_ptr<VCMTiming> timing_;  // Jitter buffer experiment.
203   VideoReceiver2 video_receiver_;
204   std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_;
205   RtpVideoStreamReceiver2 rtp_video_stream_receiver_;
206   std::unique_ptr<VideoStreamDecoder> video_stream_decoder_;
207   RtpStreamsSynchronizer rtp_stream_sync_;
208 
209   // TODO(nisse, philipel): Creation and ownership of video encoders should be
210   // moved to the new VideoStreamDecoder.
211   std::vector<std::unique_ptr<VideoDecoder>> video_decoders_;
212 
213   // Members for the new jitter buffer experiment.
214   std::unique_ptr<video_coding::FrameBuffer> frame_buffer_;
215 
216   std::unique_ptr<RtpStreamReceiverInterface> media_receiver_;
217   std::unique_ptr<RtxReceiveStream> rtx_receive_stream_;
218   std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_;
219 
220   // Whenever we are in an undecodable state (stream has just started or due to
221   // a decoding error) we require a keyframe to restart the stream.
222   bool keyframe_required_ RTC_GUARDED_BY(decode_queue_) = true;
223 
224   // If we have successfully decoded any frame.
225   bool frame_decoded_ RTC_GUARDED_BY(decode_queue_) = false;
226 
227   int64_t last_keyframe_request_ms_ RTC_GUARDED_BY(decode_queue_) = 0;
228   int64_t last_complete_frame_time_ms_
229       RTC_GUARDED_BY(worker_sequence_checker_) = 0;
230 
231   // Keyframe request intervals are configurable through field trials.
232   const int max_wait_for_keyframe_ms_;
233   const int max_wait_for_frame_ms_;
234 
235   // All of them tries to change current min_playout_delay on |timing_| but
236   // source of the change request is different in each case. Among them the
237   // biggest delay is used. -1 means use default value from the |timing_|.
238   //
239   // Minimum delay as decided by the RTP playout delay extension.
240   int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) =
241       -1;
242   // Minimum delay as decided by the setLatency function in "webrtc/api".
243   int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) =
244       -1;
245   // Minimum delay as decided by the A/V synchronization feature.
246   int syncable_minimum_playout_delay_ms_
247       RTC_GUARDED_BY(worker_sequence_checker_) = -1;
248 
249   // Maximum delay as decided by the RTP playout delay extension.
250   int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) =
251       -1;
252 
253   // Function that is triggered with encoded frames, if not empty.
254   std::function<void(const RecordableEncodedFrame&)>
255       encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_);
256   // Set to true while we're requesting keyframes but not yet received one.
257   bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) =
258       false;
259 
260   // Defined last so they are destroyed before all other members.
261   rtc::TaskQueue decode_queue_;
262 
263   // Used to signal destruction to potentially pending tasks.
264   ScopedTaskSafety task_safety_;
265 };
266 }  // namespace internal
267 }  // namespace webrtc
268 
269 #endif  // VIDEO_VIDEO_RECEIVE_STREAM2_H_
270