1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 12 #define MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <utility> 18 19 #include "modules/video_coding/frame_object.h" 20 #include "modules/video_coding/include/video_coding_defines.h" 21 #include "modules/video_coding/inter_frame_delay.h" 22 #include "rtc_base/constructormagic.h" 23 #include "rtc_base/criticalsection.h" 24 #include "rtc_base/event.h" 25 #include "rtc_base/numerics/sequence_number_util.h" 26 #include "rtc_base/thread_annotations.h" 27 28 namespace webrtc { 29 30 class Clock; 31 class VCMReceiveStatisticsCallback; 32 class VCMJitterEstimator; 33 class VCMTiming; 34 35 namespace video_coding { 36 37 class FrameBuffer { 38 public: 39 enum ReturnReason { kFrameFound, kTimeout, kStopped }; 40 41 FrameBuffer(Clock* clock, 42 VCMJitterEstimator* jitter_estimator, 43 VCMTiming* timing, 44 VCMReceiveStatisticsCallback* stats_proxy); 45 46 virtual ~FrameBuffer(); 47 48 // Insert a frame into the frame buffer. Returns the picture id 49 // of the last continuous frame or -1 if there is no continuous frame. 50 int InsertFrame(std::unique_ptr<FrameObject> frame); 51 52 // Get the next frame for decoding. Will return at latest after 53 // |max_wait_time_ms|. 54 // - If a frame is available within |max_wait_time_ms| it will return 55 // kFrameFound and set |frame_out| to the resulting frame. 56 // - If no frame is available after |max_wait_time_ms| it will return 57 // kTimeout. 58 // - If the FrameBuffer is stopped then it will return kStopped. 59 ReturnReason NextFrame(int64_t max_wait_time_ms, 60 std::unique_ptr<FrameObject>* frame_out, 61 bool keyframe_required = false); 62 63 // Tells the FrameBuffer which protection mode that is in use. Affects 64 // the frame timing. 65 // TODO(philipel): Remove this when new timing calculations has been 66 // implemented. 67 void SetProtectionMode(VCMVideoProtection mode); 68 69 // Start the frame buffer, has no effect if the frame buffer is started. 70 // The frame buffer is started upon construction. 71 void Start(); 72 73 // Stop the frame buffer, causing any sleeping thread in NextFrame to 74 // return immediately. 75 void Stop(); 76 77 // Updates the RTT for jitter buffer estimation. 78 void UpdateRtt(int64_t rtt_ms); 79 80 private: 81 struct FrameKey { FrameKeyFrameKey82 FrameKey() : picture_id(-1), spatial_layer(0) {} FrameKeyFrameKey83 FrameKey(int64_t picture_id, uint8_t spatial_layer) 84 : picture_id(picture_id), spatial_layer(spatial_layer) {} 85 86 bool operator<(const FrameKey& rhs) const { 87 if (picture_id == rhs.picture_id) 88 return spatial_layer < rhs.spatial_layer; 89 return picture_id < rhs.picture_id; 90 } 91 92 bool operator<=(const FrameKey& rhs) const { return !(rhs < *this); } 93 94 int64_t picture_id; 95 uint8_t spatial_layer; 96 }; 97 98 struct FrameInfo { 99 // The maximum number of frames that can depend on this frame. 100 static constexpr size_t kMaxNumDependentFrames = 8; 101 102 // Which other frames that have direct unfulfilled dependencies 103 // on this frame. 104 // TODO(philipel): Add simple modify/access functions to prevent adding too 105 // many |dependent_frames|. 106 FrameKey dependent_frames[kMaxNumDependentFrames]; 107 size_t num_dependent_frames = 0; 108 109 // A frame is continiuous if it has all its referenced/indirectly 110 // referenced frames. 111 // 112 // How many unfulfilled frames this frame have until it becomes continuous. 113 size_t num_missing_continuous = 0; 114 115 // A frame is decodable if all its referenced frames have been decoded. 116 // 117 // How many unfulfilled frames this frame have until it becomes decodable. 118 size_t num_missing_decodable = 0; 119 120 // If this frame is continuous or not. 121 bool continuous = false; 122 123 // The actual FrameObject. 124 std::unique_ptr<FrameObject> frame; 125 }; 126 127 using FrameMap = std::map<FrameKey, FrameInfo>; 128 129 // Check that the references of |frame| are valid. 130 bool ValidReferences(const FrameObject& frame) const; 131 132 // Updates the minimal and maximal playout delays 133 // depending on the frame. 134 void UpdatePlayoutDelays(const FrameObject& frame) 135 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 136 137 // Update all directly dependent and indirectly dependent frames and mark 138 // them as continuous if all their references has been fulfilled. 139 void PropagateContinuity(FrameMap::iterator start) 140 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 141 142 // Marks the frame as decoded and updates all directly dependent frames. 143 void PropagateDecodability(const FrameInfo& info) 144 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 145 146 // Advances |last_decoded_frame_it_| to |decoded| and removes old 147 // frame info. 148 void AdvanceLastDecodedFrame(FrameMap::iterator decoded) 149 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 150 151 // Update the corresponding FrameInfo of |frame| and all FrameInfos that 152 // |frame| references. 153 // Return false if |frame| will never be decodable, true otherwise. 154 bool UpdateFrameInfoWithIncomingFrame(const FrameObject& frame, 155 FrameMap::iterator info) 156 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 157 158 void UpdateJitterDelay() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 159 160 void UpdateTimingFrameInfo() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 161 162 void ClearFramesAndHistory() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 163 164 bool HasBadRenderTiming(const FrameObject& frame, int64_t now_ms) 165 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); 166 167 FrameMap frames_ RTC_GUARDED_BY(crit_); 168 169 rtc::CriticalSection crit_; 170 Clock* const clock_; 171 rtc::Event new_continuous_frame_event_; 172 VCMJitterEstimator* const jitter_estimator_ RTC_GUARDED_BY(crit_); 173 VCMTiming* const timing_ RTC_GUARDED_BY(crit_); 174 VCMInterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(crit_); 175 uint32_t last_decoded_frame_timestamp_ RTC_GUARDED_BY(crit_); 176 FrameMap::iterator last_decoded_frame_it_ RTC_GUARDED_BY(crit_); 177 FrameMap::iterator last_continuous_frame_it_ RTC_GUARDED_BY(crit_); 178 FrameMap::iterator next_frame_it_ RTC_GUARDED_BY(crit_); 179 int num_frames_history_ RTC_GUARDED_BY(crit_); 180 int num_frames_buffered_ RTC_GUARDED_BY(crit_); 181 bool stopped_ RTC_GUARDED_BY(crit_); 182 VCMVideoProtection protection_mode_ RTC_GUARDED_BY(crit_); 183 VCMReceiveStatisticsCallback* const stats_callback_; 184 int64_t last_log_non_decoded_ms_ RTC_GUARDED_BY(crit_); 185 186 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer); 187 }; 188 189 } // namespace video_coding 190 } // namespace webrtc 191 192 #endif // MODULES_VIDEO_CODING_FRAME_BUFFER2_H_ 193