1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_JITTER_BUFFER_H_ 12 #define MODULES_VIDEO_CODING_JITTER_BUFFER_H_ 13 14 #include <list> 15 #include <map> 16 #include <memory> 17 #include <set> 18 #include <vector> 19 20 #include "modules/include/module_common_types.h" 21 #include "modules/utility/include/process_thread.h" 22 #include "modules/video_coding/decoding_state.h" 23 #include "modules/video_coding/include/video_coding.h" 24 #include "modules/video_coding/include/video_coding_defines.h" 25 #include "modules/video_coding/inter_frame_delay.h" 26 #include "modules/video_coding/jitter_buffer_common.h" 27 #include "modules/video_coding/jitter_estimator.h" 28 #include "modules/video_coding/nack_module.h" 29 #include "rtc_base/constructormagic.h" 30 #include "rtc_base/criticalsection.h" 31 #include "rtc_base/thread_annotations.h" 32 #include "typedefs.h" // NOLINT(build/include) 33 34 namespace webrtc { 35 36 enum VCMNackMode { kNack, kNoNack }; 37 38 // forward declarations 39 class Clock; 40 class EventFactory; 41 class EventWrapper; 42 class VCMFrameBuffer; 43 class VCMPacket; 44 class VCMEncodedFrame; 45 46 typedef std::list<VCMFrameBuffer*> UnorderedFrameList; 47 48 struct VCMJitterSample { VCMJitterSampleVCMJitterSample49 VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {} 50 uint32_t timestamp; 51 uint32_t frame_size; 52 int64_t latest_packet_time; 53 }; 54 55 class TimestampLessThan { 56 public: operator()57 bool operator()(uint32_t timestamp1, uint32_t timestamp2) const { 58 return IsNewerTimestamp(timestamp2, timestamp1); 59 } 60 }; 61 62 class FrameList 63 : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> { 64 public: 65 void InsertFrame(VCMFrameBuffer* frame); 66 VCMFrameBuffer* FindFrame(uint16_t seq_num, uint32_t timestamp); 67 VCMFrameBuffer* PopFrame(uint32_t timestamp); 68 VCMFrameBuffer* Front() const; 69 VCMFrameBuffer* Back() const; 70 int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it, 71 UnorderedFrameList* free_frames); 72 void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state, 73 UnorderedFrameList* free_frames); 74 void Reset(UnorderedFrameList* free_frames); 75 }; 76 77 class Vp9SsMap { 78 public: 79 typedef std::map<uint32_t, GofInfoVP9, TimestampLessThan> SsMap; 80 bool Insert(const VCMPacket& packet); 81 void Reset(); 82 83 // Removes SS data that are older than |timestamp|. 84 // The |timestamp| should be an old timestamp, i.e. packets with older 85 // timestamps should no longer be inserted. 86 void RemoveOld(uint32_t timestamp); 87 88 bool UpdatePacket(VCMPacket* packet); 89 void UpdateFrames(FrameList* frames); 90 91 // Public for testing. 92 // Returns an iterator to the corresponding SS data for the input |timestamp|. 93 bool Find(uint32_t timestamp, SsMap::iterator* it); 94 95 private: 96 // These two functions are called by RemoveOld. 97 // Checks if it is time to do a clean up (done each kSsCleanupIntervalSec). 98 bool TimeForCleanup(uint32_t timestamp) const; 99 100 // Advances the oldest SS data to handle timestamp wrap in cases where SS data 101 // are received very seldom (e.g. only once in beginning, second when 102 // IsNewerTimestamp is not true). 103 void AdvanceFront(uint32_t timestamp); 104 105 SsMap ss_map_; 106 }; 107 108 class VCMJitterBuffer { 109 public: 110 VCMJitterBuffer(Clock* clock, 111 std::unique_ptr<EventWrapper> event, 112 NackSender* nack_sender = nullptr, 113 KeyFrameRequestSender* keyframe_request_sender = nullptr); 114 115 ~VCMJitterBuffer(); 116 117 // Initializes and starts jitter buffer. 118 void Start(); 119 120 // Signals all internal events and stops the jitter buffer. 121 void Stop(); 122 123 // Returns true if the jitter buffer is running. 124 bool Running() const; 125 126 // Empty the jitter buffer of all its data. 127 void Flush(); 128 129 // Get the number of received frames, by type, since the jitter buffer 130 // was started. 131 FrameCounts FrameStatistics() const; 132 133 // Gets number of packets received. 134 int num_packets() const; 135 136 // Gets number of duplicated packets received. 137 int num_duplicated_packets() const; 138 139 // Gets number of packets discarded by the jitter buffer. 140 int num_discarded_packets() const; 141 142 // Statistics, Calculate frame and bit rates. 143 void IncomingRateStatistics(unsigned int* framerate, unsigned int* bitrate); 144 145 // Wait |max_wait_time_ms| for a complete frame to arrive. 146 // If found, a pointer to the frame is returned. Returns nullptr otherwise. 147 VCMEncodedFrame* NextCompleteFrame(uint32_t max_wait_time_ms); 148 149 // Locates a frame for decoding (even an incomplete) without delay. 150 // The function returns true once such a frame is found, its corresponding 151 // timestamp is returned. Otherwise, returns false. 152 bool NextMaybeIncompleteTimestamp(uint32_t* timestamp); 153 154 // Extract frame corresponding to input timestamp. 155 // Frame will be set to a decoding state. 156 VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp); 157 158 // Releases a frame returned from the jitter buffer, should be called when 159 // done with decoding. 160 void ReleaseFrame(VCMEncodedFrame* frame); 161 162 // Returns the time in ms when the latest packet was inserted into the frame. 163 // Retransmitted is set to true if any of the packets belonging to the frame 164 // has been retransmitted. 165 int64_t LastPacketTime(const VCMEncodedFrame* frame, 166 bool* retransmitted) const; 167 168 // Inserts a packet into a frame returned from GetFrame(). 169 // If the return value is <= 0, |frame| is invalidated and the pointer must 170 // be dropped after this function returns. 171 VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted); 172 173 // Returns the estimated jitter in milliseconds. 174 uint32_t EstimatedJitterMs(); 175 176 // Updates the round-trip time estimate. 177 void UpdateRtt(int64_t rtt_ms); 178 179 // Set the NACK mode. |high_rtt_nack_threshold_ms| is an RTT threshold in ms 180 // above which NACK will be disabled if the NACK mode is |kNack|, -1 meaning 181 // that NACK is always enabled in the |kNack| mode. 182 // |low_rtt_nack_threshold_ms| is an RTT threshold in ms below which we expect 183 // to rely on NACK only, and therefore are using larger buffers to have time 184 // to wait for retransmissions. 185 void SetNackMode(VCMNackMode mode, 186 int64_t low_rtt_nack_threshold_ms, 187 int64_t high_rtt_nack_threshold_ms); 188 189 void SetNackSettings(size_t max_nack_list_size, 190 int max_packet_age_to_nack, 191 int max_incomplete_time_ms); 192 193 // Returns the current NACK mode. 194 VCMNackMode nack_mode() const; 195 196 // Returns a list of the sequence numbers currently missing. 197 std::vector<uint16_t> GetNackList(bool* request_key_frame); 198 199 // Set decode error mode - Should not be changed in the middle of the 200 // session. Changes will not influence frames already in the buffer. 201 void SetDecodeErrorMode(VCMDecodeErrorMode error_mode); decode_error_mode()202 VCMDecodeErrorMode decode_error_mode() const { return decode_error_mode_; } 203 204 void RegisterStatsCallback(VCMReceiveStatisticsCallback* callback); 205 206 private: 207 class SequenceNumberLessThan { 208 public: operator()209 bool operator()(const uint16_t& sequence_number1, 210 const uint16_t& sequence_number2) const { 211 return IsNewerSequenceNumber(sequence_number2, sequence_number1); 212 } 213 }; 214 typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet; 215 216 // Gets the frame assigned to the timestamp of the packet. May recycle 217 // existing frames if no free frames are available. Returns an error code if 218 // failing, or kNoError on success. |frame_list| contains which list the 219 // packet was in, or NULL if it was not in a FrameList (a new frame). 220 VCMFrameBufferEnum GetFrame(const VCMPacket& packet, 221 VCMFrameBuffer** frame, 222 FrameList** frame_list) 223 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 224 225 // Returns true if |frame| is continuous in |decoding_state|, not taking 226 // decodable frames into account. 227 bool IsContinuousInState(const VCMFrameBuffer& frame, 228 const VCMDecodingState& decoding_state) const 229 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 230 // Returns true if |frame| is continuous in the |last_decoded_state_|, taking 231 // all decodable frames into account. 232 bool IsContinuous(const VCMFrameBuffer& frame) const 233 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 234 // Looks for frames in |incomplete_frames_| which are continuous in the 235 // provided |decoded_state|. Starts the search from the timestamp of 236 // |decoded_state|. 237 void FindAndInsertContinuousFramesWithState( 238 const VCMDecodingState& decoded_state) 239 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 240 // Looks for frames in |incomplete_frames_| which are continuous in 241 // |last_decoded_state_| taking all decodable frames into account. Starts 242 // the search from |new_frame|. 243 void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame) 244 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 245 VCMFrameBuffer* NextFrame() const RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 246 // Returns true if the NACK list was updated to cover sequence numbers up to 247 // |sequence_number|. If false a key frame is needed to get into a state where 248 // we can continue decoding. 249 bool UpdateNackList(uint16_t sequence_number) 250 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 251 bool TooLargeNackList() const; 252 // Returns true if the NACK list was reduced without problem. If false a key 253 // frame is needed to get into a state where we can continue decoding. 254 bool HandleTooLargeNackList() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 255 bool MissingTooOldPacket(uint16_t latest_sequence_number) const 256 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 257 // Returns true if the too old packets was successfully removed from the NACK 258 // list. If false, a key frame is needed to get into a state where we can 259 // continue decoding. 260 bool HandleTooOldPackets(uint16_t latest_sequence_number) 261 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 262 // Drops all packets in the NACK list up until |last_decoded_sequence_number|. 263 void DropPacketsFromNackList(uint16_t last_decoded_sequence_number); 264 265 // Gets an empty frame, creating a new frame if necessary (i.e. increases 266 // jitter buffer size). 267 VCMFrameBuffer* GetEmptyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 268 269 // Attempts to increase the size of the jitter buffer. Returns true on 270 // success, false otherwise. 271 bool TryToIncreaseJitterBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 272 273 // Recycles oldest frames until a key frame is found. Used if jitter buffer is 274 // completely full. Returns true if a key frame was found. 275 bool RecycleFramesUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 276 277 // Updates the frame statistics. 278 // Counts only complete frames, so decodable incomplete frames will not be 279 // counted. 280 void CountFrame(const VCMFrameBuffer& frame) 281 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 282 283 // Update rolling average of packets per frame. 284 void UpdateAveragePacketsPerFrame(int current_number_packets_); 285 286 // Cleans the frame list in the JB from old/empty frames. 287 // Should only be called prior to actual use. 288 void CleanUpOldOrEmptyFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 289 290 // Returns true if |packet| is likely to have been retransmitted. 291 bool IsPacketRetransmitted(const VCMPacket& packet) const; 292 293 // The following three functions update the jitter estimate with the 294 // payload size, receive time and RTP timestamp of a frame. 295 void UpdateJitterEstimate(const VCMJitterSample& sample, 296 bool incomplete_frame); 297 void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame); 298 void UpdateJitterEstimate(int64_t latest_packet_time_ms, 299 uint32_t timestamp, 300 unsigned int frame_size, 301 bool incomplete_frame); 302 303 // Returns true if we should wait for retransmissions, false otherwise. 304 bool WaitForRetransmissions(); 305 306 int NonContinuousOrIncompleteDuration() 307 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 308 309 uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const; 310 311 void UpdateHistograms() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 312 313 // Reset frame buffer and return it to free_frames_. 314 void RecycleFrameBuffer(VCMFrameBuffer* frame) 315 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 316 317 Clock* clock_; 318 // If we are running (have started) or not. 319 bool running_; 320 rtc::CriticalSection crit_sect_; 321 // Event to signal when we have a frame ready for decoder. 322 std::unique_ptr<EventWrapper> frame_event_; 323 // Number of allocated frames. 324 int max_number_of_frames_; 325 UnorderedFrameList free_frames_ RTC_GUARDED_BY(crit_sect_); 326 FrameList decodable_frames_ RTC_GUARDED_BY(crit_sect_); 327 FrameList incomplete_frames_ RTC_GUARDED_BY(crit_sect_); 328 VCMDecodingState last_decoded_state_ RTC_GUARDED_BY(crit_sect_); 329 bool first_packet_since_reset_; 330 331 // Statistics. 332 VCMReceiveStatisticsCallback* stats_callback_ RTC_GUARDED_BY(crit_sect_); 333 // Frame counts for each type (key, delta, ...) 334 FrameCounts receive_statistics_; 335 // Latest calculated frame rates of incoming stream. 336 unsigned int incoming_frame_rate_; 337 unsigned int incoming_frame_count_; 338 int64_t time_last_incoming_frame_count_; 339 unsigned int incoming_bit_count_; 340 unsigned int incoming_bit_rate_; 341 // Number of packets in a row that have been too old. 342 int num_consecutive_old_packets_; 343 // Number of packets received. 344 int num_packets_ RTC_GUARDED_BY(crit_sect_); 345 // Number of duplicated packets received. 346 int num_duplicated_packets_ RTC_GUARDED_BY(crit_sect_); 347 // Number of packets discarded by the jitter buffer. 348 int num_discarded_packets_ RTC_GUARDED_BY(crit_sect_); 349 // Time when first packet is received. 350 int64_t time_first_packet_ms_ RTC_GUARDED_BY(crit_sect_); 351 352 // Jitter estimation. 353 // Filter for estimating jitter. 354 VCMJitterEstimator jitter_estimate_; 355 // Calculates network delays used for jitter calculations. 356 VCMInterFrameDelay inter_frame_delay_; 357 VCMJitterSample waiting_for_completion_; 358 int64_t rtt_ms_; 359 360 // NACK and retransmissions. 361 VCMNackMode nack_mode_; 362 int64_t low_rtt_nack_threshold_ms_; 363 int64_t high_rtt_nack_threshold_ms_; 364 // Holds the internal NACK list (the missing sequence numbers). 365 SequenceNumberSet missing_sequence_numbers_; 366 uint16_t latest_received_sequence_number_; 367 size_t max_nack_list_size_; 368 int max_packet_age_to_nack_; // Measured in sequence numbers. 369 int max_incomplete_time_ms_; 370 371 VCMDecodeErrorMode decode_error_mode_; 372 // Estimated rolling average of packets per frame 373 float average_packets_per_frame_; 374 // average_packets_per_frame converges fast if we have fewer than this many 375 // frames. 376 int frame_counter_; 377 378 RTC_DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer); 379 }; 380 } // namespace webrtc 381 382 #endif // MODULES_VIDEO_CODING_JITTER_BUFFER_H_ 383