1 /* 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_ 12 #define VIDEO_VIDEO_RECEIVE_STREAM_H_ 13 14 #include <memory> 15 #include <vector> 16 17 #include "api/task_queue/task_queue_factory.h" 18 #include "api/video/recordable_encoded_frame.h" 19 #include "call/rtp_packet_sink_interface.h" 20 #include "call/syncable.h" 21 #include "call/video_receive_stream.h" 22 #include "modules/rtp_rtcp/include/flexfec_receiver.h" 23 #include "modules/rtp_rtcp/source/source_tracker.h" 24 #include "modules/video_coding/frame_buffer2.h" 25 #include "modules/video_coding/video_receiver2.h" 26 #include "rtc_base/synchronization/mutex.h" 27 #include "rtc_base/synchronization/sequence_checker.h" 28 #include "rtc_base/task_queue.h" 29 #include "system_wrappers/include/clock.h" 30 #include "video/receive_statistics_proxy.h" 31 #include "video/rtp_streams_synchronizer.h" 32 #include "video/rtp_video_stream_receiver.h" 33 #include "video/transport_adapter.h" 34 #include "video/video_stream_decoder.h" 35 36 namespace webrtc { 37 38 class CallStats; 39 class ProcessThread; 40 class RtpStreamReceiverInterface; 41 class RtpStreamReceiverControllerInterface; 42 class RtxReceiveStream; 43 class VCMTiming; 44 45 namespace internal { 46 47 class VideoReceiveStream : public webrtc::VideoReceiveStream, 48 public rtc::VideoSinkInterface<VideoFrame>, 49 public NackSender, 50 public video_coding::OnCompleteFrameCallback, 51 public Syncable, 52 public CallStatsObserver { 53 public: 54 // The default number of milliseconds to pass before re-requesting a key frame 55 // to be sent. 56 static constexpr int kMaxWaitForKeyFrameMs = 200; 57 58 VideoReceiveStream(TaskQueueFactory* task_queue_factory, 59 RtpStreamReceiverControllerInterface* receiver_controller, 60 int num_cpu_cores, 61 PacketRouter* packet_router, 62 VideoReceiveStream::Config config, 63 ProcessThread* process_thread, 64 CallStats* call_stats, 65 Clock* clock, 66 VCMTiming* timing); 67 VideoReceiveStream(TaskQueueFactory* task_queue_factory, 68 RtpStreamReceiverControllerInterface* receiver_controller, 69 int num_cpu_cores, 70 PacketRouter* packet_router, 71 VideoReceiveStream::Config config, 72 ProcessThread* process_thread, 73 CallStats* call_stats, 74 Clock* clock); 75 ~VideoReceiveStream() override; 76 config()77 const Config& config() const { return config_; } 78 79 void SignalNetworkState(NetworkState state); 80 bool DeliverRtcp(const uint8_t* packet, size_t length); 81 82 void SetSync(Syncable* audio_syncable); 83 84 // Implements webrtc::VideoReceiveStream. 85 void Start() override; 86 void Stop() override; 87 88 webrtc::VideoReceiveStream::Stats GetStats() const override; 89 90 void AddSecondarySink(RtpPacketSinkInterface* sink) override; 91 void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; 92 93 // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called 94 // from webrtc/api level and requested by user code. For e.g. blink/js layer 95 // in Chromium. 96 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; 97 int GetBaseMinimumPlayoutDelayMs() const override; 98 99 void SetFrameDecryptor( 100 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; 101 void SetDepacketizerToDecoderFrameTransformer( 102 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override; 103 104 // Implements rtc::VideoSinkInterface<VideoFrame>. 105 void OnFrame(const VideoFrame& video_frame) override; 106 107 // Implements NackSender. 108 // For this particular override of the interface, 109 // only (buffering_allowed == true) is acceptable. 110 void SendNack(const std::vector<uint16_t>& sequence_numbers, 111 bool buffering_allowed) override; 112 113 // Implements video_coding::OnCompleteFrameCallback. 114 void OnCompleteFrame( 115 std::unique_ptr<video_coding::EncodedFrame> frame) override; 116 117 // Implements CallStatsObserver::OnRttUpdate 118 void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; 119 120 // Implements Syncable. 121 uint32_t id() const override; 122 absl::optional<Syncable::Info> GetInfo() const override; 123 bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, 124 int64_t* time_ms) const override; 125 void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, 126 int64_t time_ms) override; 127 128 // SetMinimumPlayoutDelay is only called by A/V sync. 129 bool SetMinimumPlayoutDelay(int delay_ms) override; 130 131 std::vector<webrtc::RtpSource> GetSources() const override; 132 133 RecordingState SetAndGetRecordingState(RecordingState state, 134 bool generate_key_frame) override; 135 void GenerateKeyFrame() override; 136 137 private: 138 int64_t GetWaitMs() const; 139 void StartNextDecode() RTC_RUN_ON(decode_queue_); 140 void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame) 141 RTC_RUN_ON(decode_queue_); 142 void HandleFrameBufferTimeout() RTC_RUN_ON(decode_queue_); 143 void UpdatePlayoutDelays() const 144 RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); 145 void RequestKeyFrame(int64_t timestamp_ms) RTC_RUN_ON(decode_queue_); 146 void HandleKeyFrameGeneration(bool received_frame_is_keyframe, int64_t now_ms) 147 RTC_RUN_ON(decode_queue_); 148 bool IsReceivingKeyFrame(int64_t timestamp_ms) const 149 RTC_RUN_ON(decode_queue_); 150 151 void UpdateHistograms(); 152 153 SequenceChecker worker_sequence_checker_; 154 SequenceChecker module_process_sequence_checker_; 155 SequenceChecker network_sequence_checker_; 156 157 TaskQueueFactory* const task_queue_factory_; 158 159 TransportAdapter transport_adapter_; 160 const VideoReceiveStream::Config config_; 161 const int num_cpu_cores_; 162 ProcessThread* const process_thread_; 163 Clock* const clock_; 164 165 CallStats* const call_stats_; 166 167 bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; 168 bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; 169 170 SourceTracker source_tracker_; 171 ReceiveStatisticsProxy stats_proxy_; 172 // Shared by media and rtx stream receivers, since the latter has no RtpRtcp 173 // module of its own. 174 const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; 175 176 std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. 177 VideoReceiver2 video_receiver_; 178 std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; 179 RtpVideoStreamReceiver rtp_video_stream_receiver_; 180 std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; 181 RtpStreamsSynchronizer rtp_stream_sync_; 182 183 // TODO(nisse, philipel): Creation and ownership of video encoders should be 184 // moved to the new VideoStreamDecoder. 185 std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; 186 187 // Members for the new jitter buffer experiment. 188 std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; 189 190 std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; 191 std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; 192 std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; 193 194 // Whenever we are in an undecodable state (stream has just started or due to 195 // a decoding error) we require a keyframe to restart the stream. 196 bool keyframe_required_ = true; 197 198 // If we have successfully decoded any frame. 199 bool frame_decoded_ = false; 200 201 int64_t last_keyframe_request_ms_ = 0; 202 int64_t last_complete_frame_time_ms_ = 0; 203 204 // Keyframe request intervals are configurable through field trials. 205 const int max_wait_for_keyframe_ms_; 206 const int max_wait_for_frame_ms_; 207 208 mutable Mutex playout_delay_lock_; 209 210 // All of them tries to change current min_playout_delay on |timing_| but 211 // source of the change request is different in each case. Among them the 212 // biggest delay is used. -1 means use default value from the |timing_|. 213 // 214 // Minimum delay as decided by the RTP playout delay extension. 215 int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 216 // Minimum delay as decided by the setLatency function in "webrtc/api". 217 int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 218 // Minimum delay as decided by the A/V synchronization feature. 219 int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = 220 -1; 221 222 // Maximum delay as decided by the RTP playout delay extension. 223 int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; 224 225 // Function that is triggered with encoded frames, if not empty. 226 std::function<void(const RecordableEncodedFrame&)> 227 encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); 228 // Set to true while we're requesting keyframes but not yet received one. 229 bool keyframe_generation_requested_ RTC_GUARDED_BY(decode_queue_) = false; 230 231 // Defined last so they are destroyed before all other members. 232 rtc::TaskQueue decode_queue_; 233 }; 234 } // namespace internal 235 } // namespace webrtc 236 237 #endif // VIDEO_VIDEO_RECEIVE_STREAM_H_ 238