1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM2_H_ 12 #define VIDEO_VIDEO_RECEIVE_STREAM2_H_ 13 14 #include <memory> 15 #include <vector> 16 17 #include "api/task_queue/task_queue_factory.h" 18 #include "api/units/timestamp.h" 19 #include "api/video/recordable_encoded_frame.h" 20 #include "call/rtp_packet_sink_interface.h" 21 #include "call/syncable.h" 22 #include "call/video_receive_stream.h" 23 #include "modules/rtp_rtcp/include/flexfec_receiver.h" 24 #include "modules/rtp_rtcp/source/source_tracker.h" 25 #include "modules/video_coding/frame_buffer2.h" 26 #include "modules/video_coding/video_receiver2.h" 27 #include "rtc_base/synchronization/sequence_checker.h" 28 #include "rtc_base/task_queue.h" 29 #include "rtc_base/task_utils/pending_task_safety_flag.h" 30 #include "system_wrappers/include/clock.h" 31 #include "video/receive_statistics_proxy2.h" 32 #include "video/rtp_streams_synchronizer2.h" 33 #include "video/rtp_video_stream_receiver2.h" 34 #include "video/transport_adapter.h" 35 #include "video/video_stream_decoder2.h" 36 37 namespace webrtc { 38 39 class ProcessThread; 40 class RtpStreamReceiverInterface; 41 class RtpStreamReceiverControllerInterface; 42 class RtxReceiveStream; 43 class VCMTiming; 44 45 namespace internal { 46 47 class CallStats; 48 49 // Utility struct for grabbing metadata from a VideoFrame and processing it 50 // asynchronously without needing the actual frame data. 51 // Additionally the caller can bundle information from the current clock 52 // when the metadata is captured, for accurate reporting and not needeing 53 // multiple calls to clock->Now(). 54 struct VideoFrameMetaData { VideoFrameMetaDataVideoFrameMetaData55 VideoFrameMetaData(const webrtc::VideoFrame& frame, Timestamp now) 56 : rtp_timestamp(frame.timestamp()), 57 timestamp_us(frame.timestamp_us()), 58 ntp_time_ms(frame.ntp_time_ms()), 59 width(frame.width()), 60 height(frame.height()), 61 decode_timestamp(now) {} 62 render_time_msVideoFrameMetaData63 int64_t render_time_ms() const { 64 return timestamp_us / rtc::kNumMicrosecsPerMillisec; 65 } 66 67 const uint32_t rtp_timestamp; 68 const int64_t timestamp_us; 69 const int64_t ntp_time_ms; 70 const int width; 71 const int height; 72 73 const Timestamp decode_timestamp; 74 }; 75 76 class VideoReceiveStream2 : public webrtc::VideoReceiveStream, 77 public rtc::VideoSinkInterface<VideoFrame>, 78 public NackSender, 79 public video_coding::OnCompleteFrameCallback, 80 public Syncable, 81 public CallStatsObserver { 82 public: 83 // The default number of milliseconds to pass before re-requesting a key frame 84 // to be sent. 85 static constexpr int kMaxWaitForKeyFrameMs = 200; 86 87 VideoReceiveStream2(TaskQueueFactory* task_queue_factory, 88 TaskQueueBase* current_queue, 89 RtpStreamReceiverControllerInterface* receiver_controller, 90 int num_cpu_cores, 91 PacketRouter* packet_router, 92 VideoReceiveStream::Config config, 93 ProcessThread* process_thread, 94 CallStats* call_stats, 95 Clock* clock, 96 VCMTiming* timing); 97 ~VideoReceiveStream2() override; 98 config()99 const Config& config() const { return config_; } 100 101 void SignalNetworkState(NetworkState state); 102 bool DeliverRtcp(const uint8_t* packet, size_t length); 103 104 void SetSync(Syncable* audio_syncable); 105 106 // Implements webrtc::VideoReceiveStream. 107 void Start() override; 108 void Stop() override; 109 110 webrtc::VideoReceiveStream::Stats GetStats() const override; 111 112 void AddSecondarySink(RtpPacketSinkInterface* sink) override; 113 void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; 114 115 // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called 116 // from webrtc/api level and requested by user code. For e.g. blink/js layer 117 // in Chromium. 118 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 119 int GetBaseMinimumPlayoutDelayMs() const override; 120 121 void SetFrameDecryptor( 122 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 123 void SetDepacketizerToDecoderFrameTransformer( 124 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; 125 126 // Implements rtc::VideoSinkInterface<VideoFrame>. 127 void OnFrame(const VideoFrame& video_frame) override; 128 129 // Implements NackSender. 130 // For this particular override of the interface, 131 // only (buffering_allowed == true) is acceptable. 132 void SendNack(const std::vector<uint16_t>& sequence_numbers, 133 bool buffering_allowed) override; 134 135 // Implements video_coding::OnCompleteFrameCallback. 136 void OnCompleteFrame( 137 std::unique_ptr<video_coding::EncodedFrame> frame) override; 138 139 // Implements CallStatsObserver::OnRttUpdate 140 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; 141 142 // Implements Syncable. 143 uint32_t id() const override; 144 absl::optional<Syncable::Info> GetInfo() const override; 145 bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, 146 int64_t* time_ms) const override; 147 void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, 148 int64_t time_ms) override; 149 150 // SetMinimumPlayoutDelay is only called by A/V sync. 151 bool SetMinimumPlayoutDelay(int delay_ms) override; 152 153 std::vector<webrtc::RtpSource> GetSources() const override; 154 155 RecordingState SetAndGetRecordingState(RecordingState state, 156 bool generate_key_frame) override; 157 void GenerateKeyFrame() override; 158 159 private: 160 int64_t GetMaxWaitMs() const RTC_RUN_ON(decode_queue_); 161 void StartNextDecode() RTC_RUN_ON(decode_queue_); 162 void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame) 163 RTC_RUN_ON(decode_queue_); 164 void HandleFrameBufferTimeout(int64_t now_ms, int64_t wait_ms) 165 RTC_RUN_ON(worker_sequence_checker_); 166 void UpdatePlayoutDelays() const 167 RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_sequence_checker_); 168 void RequestKeyFrame(int64_t timestamp_ms) 169 RTC_RUN_ON(worker_sequence_checker_); 170 void HandleKeyFrameGeneration(bool received_frame_is_keyframe, 171 int64_t now_ms, 172 bool always_request_key_frame, 173 bool keyframe_request_is_due) 174 RTC_RUN_ON(worker_sequence_checker_); 175 bool IsReceivingKeyFrame(int64_t timestamp_ms) const 176 RTC_RUN_ON(worker_sequence_checker_); 177 178 void UpdateHistograms(); 179 180 SequenceChecker worker_sequence_checker_; 181 SequenceChecker module_process_sequence_checker_; 182 183 TaskQueueFactory* const task_queue_factory_; 184 185 TransportAdapter transport_adapter_; 186 const VideoReceiveStream::Config config_; 187 const int num_cpu_cores_; 188 TaskQueueBase* const worker_thread_; 189 Clock* const clock_; 190 191 CallStats* const call_stats_; 192 193 bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; 194 bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; 195 196 SourceTracker source_tracker_; 197 ReceiveStatisticsProxy stats_proxy_; 198 // Shared by media and rtx stream receivers, since the latter has no RtpRtcp 199 // module of its own. 200 const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 201 202 std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. 203 VideoReceiver2 video_receiver_; 204 std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; 205 RtpVideoStreamReceiver2 rtp_video_stream_receiver_; 206 std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; 207 RtpStreamsSynchronizer rtp_stream_sync_; 208 209 // TODO(nisse, philipel): Creation and ownership of video encoders should be 210 // moved to the new VideoStreamDecoder. 211 std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; 212 213 // Members for the new jitter buffer experiment. 214 std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; 215 216 std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; 217 std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; 218 std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; 219 220 // Whenever we are in an undecodable state (stream has just started or due to 221 // a decoding error) we require a keyframe to restart the stream. 222 bool keyframe_required_ RTC_GUARDED_BY(decode_queue_) = true; 223 224 // If we have successfully decoded any frame. 225 bool frame_decoded_ RTC_GUARDED_BY(decode_queue_) = false; 226 227 int64_t last_keyframe_request_ms_ RTC_GUARDED_BY(decode_queue_) = 0; 228 int64_t last_complete_frame_time_ms_ 229 RTC_GUARDED_BY(worker_sequence_checker_) = 0; 230 231 // Keyframe request intervals are configurable through field trials. 232 const int max_wait_for_keyframe_ms_; 233 const int max_wait_for_frame_ms_; 234 235 // All of them tries to change current min_playout_delay on |timing_| but 236 // source of the change request is different in each case. Among them the 237 // biggest delay is used. -1 means use default value from the |timing_|. 238 // 239 // Minimum delay as decided by the RTP playout delay extension. 240 int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 241 -1; 242 // Minimum delay as decided by the setLatency function in "webrtc/api". 243 int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 244 -1; 245 // Minimum delay as decided by the A/V synchronization feature. 246 int syncable_minimum_playout_delay_ms_ 247 RTC_GUARDED_BY(worker_sequence_checker_) = -1; 248 249 // Maximum delay as decided by the RTP playout delay extension. 250 int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(worker_sequence_checker_) = 251 -1; 252 253 // Function that is triggered with encoded frames, if not empty. 254 std::function<void(const RecordableEncodedFrame&)> 255 encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); 256 // Set to true while we're requesting keyframes but not yet received one. 257 bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) = 258 false; 259 260 // Defined last so they are destroyed before all other members. 261 rtc::TaskQueue decode_queue_; 262 263 // Used to signal destruction to potentially pending tasks. 264 ScopedTaskSafety task_safety_; 265 }; 266 } // namespace internal 267 } // namespace webrtc 268 269 #endif // VIDEO_VIDEO_RECEIVE_STREAM2_H_ 270