1 /*
2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_
12 #define VIDEO_VIDEO_RECEIVE_STREAM_H_
13 
14 #include <memory>
15 #include <vector>
16 
17 #include "api/task_queue/task_queue_factory.h"
18 #include "api/transport/media/media_transport_interface.h"
19 #include "api/video/recordable_encoded_frame.h"
20 #include "call/rtp_packet_sink_interface.h"
21 #include "call/syncable.h"
22 #include "call/video_receive_stream.h"
23 #include "modules/rtp_rtcp/include/flexfec_receiver.h"
24 #include "modules/rtp_rtcp/source/source_tracker.h"
25 #include "modules/video_coding/frame_buffer2.h"
26 #include "modules/video_coding/video_receiver2.h"
27 #include "rtc_base/synchronization/sequence_checker.h"
28 #include "rtc_base/task_queue.h"
29 #include "system_wrappers/include/clock.h"
30 #include "video/receive_statistics_proxy.h"
31 #include "video/rtp_streams_synchronizer.h"
32 #include "video/rtp_video_stream_receiver.h"
33 #include "video/transport_adapter.h"
34 #include "video/video_stream_decoder.h"
35 
36 namespace webrtc {
37 
38 class CallStats;
39 class ProcessThread;
40 class RTPFragmentationHeader;
41 class RtpStreamReceiverInterface;
42 class RtpStreamReceiverControllerInterface;
43 class RtxReceiveStream;
44 class VCMTiming;
45 
46 namespace internal {
47 
48 class VideoReceiveStream : public webrtc::VideoReceiveStream,
49                            public rtc::VideoSinkInterface<VideoFrame>,
50                            public NackSender,
51                            public video_coding::OnCompleteFrameCallback,
52                            public Syncable,
53                            public CallStatsObserver {
54  public:
55   // The default number of milliseconds to pass before re-requesting a key frame
56   // to be sent.
57   static constexpr int kMaxWaitForKeyFrameMs = 200;
58 
59   VideoReceiveStream(TaskQueueFactory* task_queue_factory,
60                      RtpStreamReceiverControllerInterface* receiver_controller,
61                      int num_cpu_cores,
62                      PacketRouter* packet_router,
63                      VideoReceiveStream::Config config,
64                      ProcessThread* process_thread,
65                      CallStats* call_stats,
66                      Clock* clock,
67                      VCMTiming* timing);
68   VideoReceiveStream(TaskQueueFactory* task_queue_factory,
69                      RtpStreamReceiverControllerInterface* receiver_controller,
70                      int num_cpu_cores,
71                      PacketRouter* packet_router,
72                      VideoReceiveStream::Config config,
73                      ProcessThread* process_thread,
74                      CallStats* call_stats,
75                      Clock* clock);
76   ~VideoReceiveStream() override;
77 
config()78   const Config& config() const { return config_; }
79 
80   void SignalNetworkState(NetworkState state);
81   bool DeliverRtcp(const uint8_t* packet, size_t length);
82 
83   void SetSync(Syncable* audio_syncable);
84 
85   // Implements webrtc::VideoReceiveStream.
86   void Start() override;
87   void Stop() override;
88 
89   webrtc::VideoReceiveStream::Stats GetStats() const override;
90 
91   void AddSecondarySink(RtpPacketSinkInterface* sink) override;
92   void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override;
93 
94   // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called
95   // from webrtc/api level and requested by user code. For e.g. blink/js layer
96   // in Chromium.
97   bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
98   int GetBaseMinimumPlayoutDelayMs() const override;
99 
100   void SetFrameDecryptor(
101       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override;
102   void SetDepacketizerToDecoderFrameTransformer(
103       rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override;
104 
105   // Implements rtc::VideoSinkInterface<VideoFrame>.
106   void OnFrame(const VideoFrame& video_frame) override;
107 
108   // Implements NackSender.
109   // For this particular override of the interface,
110   // only (buffering_allowed == true) is acceptable.
111   void SendNack(const std::vector<uint16_t>& sequence_numbers,
112                 bool buffering_allowed) override;
113 
114   // Implements video_coding::OnCompleteFrameCallback.
115   void OnCompleteFrame(
116       std::unique_ptr<video_coding::EncodedFrame> frame) override;
117 
118   // Implements CallStatsObserver::OnRttUpdate
119   void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override;
120 
121   // Implements Syncable.
122   uint32_t id() const override;
123   absl::optional<Syncable::Info> GetInfo() const override;
124   bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
125                               int64_t* time_ms) const override;
126   void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,
127                                          int64_t time_ms) override;
128 
129   // SetMinimumPlayoutDelay is only called by A/V sync.
130   void SetMinimumPlayoutDelay(int delay_ms) override;
131 
132   std::vector<webrtc::RtpSource> GetSources() const override;
133 
134   RecordingState SetAndGetRecordingState(RecordingState state,
135                                          bool generate_key_frame) override;
136   void GenerateKeyFrame() override;
137 
138  private:
139   int64_t GetWaitMs() const;
140   void StartNextDecode() RTC_RUN_ON(decode_queue_);
141   void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame)
142       RTC_RUN_ON(decode_queue_);
143   void HandleFrameBufferTimeout() RTC_RUN_ON(decode_queue_);
144   void UpdatePlayoutDelays() const
145       RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_);
146   void RequestKeyFrame(int64_t timestamp_ms) RTC_RUN_ON(decode_queue_);
147   void HandleKeyFrameGeneration(bool received_frame_is_keyframe, int64_t now_ms)
148       RTC_RUN_ON(decode_queue_);
149   bool IsReceivingKeyFrame(int64_t timestamp_ms) const
150       RTC_RUN_ON(decode_queue_);
151 
152   void UpdateHistograms();
153 
154   SequenceChecker worker_sequence_checker_;
155   SequenceChecker module_process_sequence_checker_;
156   SequenceChecker network_sequence_checker_;
157 
158   TaskQueueFactory* const task_queue_factory_;
159 
160   TransportAdapter transport_adapter_;
161   const VideoReceiveStream::Config config_;
162   const int num_cpu_cores_;
163   ProcessThread* const process_thread_;
164   Clock* const clock_;
165 
166   CallStats* const call_stats_;
167 
168   bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false;
169   bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true;
170 
171   SourceTracker source_tracker_;
172   ReceiveStatisticsProxy stats_proxy_;
173   // Shared by media and rtx stream receivers, since the latter has no RtpRtcp
174   // module of its own.
175   const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
176 
177   std::unique_ptr<VCMTiming> timing_;  // Jitter buffer experiment.
178   VideoReceiver2 video_receiver_;
179   std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_;
180   RtpVideoStreamReceiver rtp_video_stream_receiver_;
181   std::unique_ptr<VideoStreamDecoder> video_stream_decoder_;
182   RtpStreamsSynchronizer rtp_stream_sync_;
183 
184   // TODO(nisse, philipel): Creation and ownership of video encoders should be
185   // moved to the new VideoStreamDecoder.
186   std::vector<std::unique_ptr<VideoDecoder>> video_decoders_;
187 
188   // Members for the new jitter buffer experiment.
189   std::unique_ptr<video_coding::FrameBuffer> frame_buffer_;
190 
191   std::unique_ptr<RtpStreamReceiverInterface> media_receiver_;
192   std::unique_ptr<RtxReceiveStream> rtx_receive_stream_;
193   std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_;
194 
195   // Whenever we are in an undecodable state (stream has just started or due to
196   // a decoding error) we require a keyframe to restart the stream.
197   bool keyframe_required_ = true;
198 
199   // If we have successfully decoded any frame.
200   bool frame_decoded_ = false;
201 
202   int64_t last_keyframe_request_ms_ = 0;
203   int64_t last_complete_frame_time_ms_ = 0;
204 
205   // Keyframe request intervals are configurable through field trials.
206   const int max_wait_for_keyframe_ms_;
207   const int max_wait_for_frame_ms_;
208 
209   rtc::CriticalSection playout_delay_lock_;
210 
211   // All of them tries to change current min_playout_delay on |timing_| but
212   // source of the change request is different in each case. Among them the
213   // biggest delay is used. -1 means use default value from the |timing_|.
214   //
215   // Minimum delay as decided by the RTP playout delay extension.
216   int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1;
217   // Minimum delay as decided by the setLatency function in "webrtc/api".
218   int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1;
219   // Minimum delay as decided by the A/V synchronization feature.
220   int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) =
221       -1;
222 
223   // Maximum delay as decided by the RTP playout delay extension.
224   int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1;
225 
226   // Function that is triggered with encoded frames, if not empty.
227   std::function<void(const RecordableEncodedFrame&)>
228       encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_);
229   // Set to true while we're requesting keyframes but not yet received one.
230   bool keyframe_generation_requested_ RTC_GUARDED_BY(decode_queue_) = false;
231 
232   // Defined last so they are destroyed before all other members.
233   rtc::TaskQueue decode_queue_;
234 };
235 }  // namespace internal
236 }  // namespace webrtc
237 
238 #endif  // VIDEO_VIDEO_RECEIVE_STREAM_H_
239