1 /* 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ 12 #define WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ 13 14 #include <array> 15 #include <map> 16 #include <memory> 17 #include <deque> 18 #include <set> 19 #include <utility> 20 21 #include "webrtc/base/criticalsection.h" 22 #include "webrtc/base/thread_annotations.h" 23 #include "webrtc/modules/include/module_common_types.h" 24 #include "webrtc/modules/video_coding/sequence_number_util.h" 25 26 namespace webrtc { 27 namespace video_coding { 28 29 class FrameObject; 30 class RtpFrameObject; 31 32 // A complete frame is a frame which has received all its packets and all its 33 // references are known. 34 class OnCompleteFrameCallback { 35 public: ~OnCompleteFrameCallback()36 virtual ~OnCompleteFrameCallback() {} 37 virtual void OnCompleteFrame(std::unique_ptr<FrameObject> frame) = 0; 38 }; 39 40 class RtpFrameReferenceFinder { 41 public: 42 explicit RtpFrameReferenceFinder(OnCompleteFrameCallback* frame_callback); 43 44 // Manage this frame until: 45 // - We have all information needed to determine its references, after 46 // which |frame_callback_| is called with the completed frame, or 47 // - We have too many stashed frames (determined by |kMaxStashedFrames) 48 // so we drop this frame, or 49 // - It gets cleared by ClearTo, which also means we drop it. 50 void ManageFrame(std::unique_ptr<RtpFrameObject> frame); 51 52 // Notifies that padding has been received, which the reference finder 53 // might need to calculate the references of a frame. 54 void PaddingReceived(uint16_t seq_num); 55 56 // Clear all stashed frames that include packets older than |seq_num|. 57 void ClearTo(uint16_t seq_num); 58 59 private: 60 static const uint16_t kPicIdLength = 1 << 15; 61 static const uint8_t kMaxTemporalLayers = 5; 62 static const int kMaxLayerInfo = 50; 63 static const int kMaxStashedFrames = 50; 64 static const int kMaxNotYetReceivedFrames = 100; 65 static const int kMaxGofSaved = 50; 66 static const int kMaxPaddingAge = 100; 67 68 69 struct GofInfo { GofInfoGofInfo70 GofInfo(GofInfoVP9* gof, uint16_t last_picture_id) 71 : gof(gof), last_picture_id(last_picture_id) {} 72 GofInfoVP9* gof; 73 uint16_t last_picture_id; 74 }; 75 76 rtc::CriticalSection crit_; 77 78 // Find the relevant group of pictures and update its "last-picture-id-with 79 // padding" sequence number. 80 void UpdateLastPictureIdWithPadding(uint16_t seq_num) 81 EXCLUSIVE_LOCKS_REQUIRED(crit_); 82 83 // Retry finding references for all frames that previously didn't have 84 // all information needed. 85 void RetryStashedFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_); 86 87 // Find references for generic frames. If |picture_id| is unspecified 88 // then packet sequence numbers will be used to determine the references 89 // of the frames. 90 void ManageFrameGeneric(std::unique_ptr<RtpFrameObject> frame, 91 int picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_); 92 93 // Find references for Vp8 frames 94 void ManageFrameVp8(std::unique_ptr<RtpFrameObject> frame) 95 EXCLUSIVE_LOCKS_REQUIRED(crit_); 96 97 // Updates all necessary state used to determine frame references 98 // for Vp8 and then calls the |frame_callback| callback with the 99 // completed frame. 100 void CompletedFrameVp8(std::unique_ptr<RtpFrameObject> frame) 101 EXCLUSIVE_LOCKS_REQUIRED(crit_); 102 103 // Find references for Vp9 frames 104 void ManageFrameVp9(std::unique_ptr<RtpFrameObject> frame) 105 EXCLUSIVE_LOCKS_REQUIRED(crit_); 106 107 // Unwrap the picture id and the frame references and then call the 108 // |frame_callback| callback with the completed frame. 109 void CompletedFrameVp9(std::unique_ptr<RtpFrameObject> frame) 110 EXCLUSIVE_LOCKS_REQUIRED(crit_); 111 112 // Check if we are missing a frame necessary to determine the references 113 // for this frame. 114 bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info) 115 EXCLUSIVE_LOCKS_REQUIRED(crit_); 116 117 // Updates which frames that have been received. If there is a gap, 118 // missing frames will be added to |missing_frames_for_layer_| or 119 // if this is an already missing frame then it will be removed. 120 void FrameReceivedVp9(uint16_t picture_id, GofInfo* info) 121 EXCLUSIVE_LOCKS_REQUIRED(crit_); 122 123 // Check if there is a frame with the up-switch flag set in the interval 124 // (|pid_ref|, |picture_id|) with temporal layer smaller than |temporal_idx|. 125 bool UpSwitchInIntervalVp9(uint16_t picture_id, 126 uint8_t temporal_idx, 127 uint16_t pid_ref) EXCLUSIVE_LOCKS_REQUIRED(crit_); 128 129 // All picture ids are unwrapped to 16 bits. 130 uint16_t UnwrapPictureId(uint16_t picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_); 131 132 // Returns true if the frame is old and should be dropped. 133 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be 134 // around M59). 135 bool Vp9PidTl0Fix(const RtpFrameObject& frame, 136 int16_t* picture_id, 137 int16_t* tl0_pic_idx) EXCLUSIVE_LOCKS_REQUIRED(crit_); 138 139 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be 140 // around M59). 141 bool DetectVp9PicIdJump(int fixed_pid, 142 int fixed_tl0, 143 uint32_t timestamp) const 144 EXCLUSIVE_LOCKS_REQUIRED(crit_); 145 146 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be 147 // around M59). 148 bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const 149 EXCLUSIVE_LOCKS_REQUIRED(crit_); 150 151 // For every group of pictures, hold two sequence numbers. The first being 152 // the sequence number of the last packet of the last completed frame, and 153 // the second being the sequence number of the last packet of the last 154 // completed frame advanced by any potential continuous packets of padding. 155 std::map<uint16_t, 156 std::pair<uint16_t, uint16_t>, 157 DescendingSeqNumComp<uint16_t>> 158 last_seq_num_gop_ GUARDED_BY(crit_); 159 160 // Save the last picture id in order to detect when there is a gap in frames 161 // that have not yet been fully received. 162 int last_picture_id_ GUARDED_BY(crit_); 163 164 // Padding packets that have been received but that are not yet continuous 165 // with any group of pictures. 166 std::set<uint16_t, DescendingSeqNumComp<uint16_t>> stashed_padding_ 167 GUARDED_BY(crit_); 168 169 // The last unwrapped picture id. Used to unwrap the picture id from a length 170 // of |kPicIdLength| to 16 bits. 171 int last_unwrap_ GUARDED_BY(crit_); 172 173 // Frames earlier than the last received frame that have not yet been 174 // fully received. 175 std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>> 176 not_yet_received_frames_ GUARDED_BY(crit_); 177 178 // Frames that have been fully received but didn't have all the information 179 // needed to determine their references. 180 std::deque<std::unique_ptr<RtpFrameObject>> stashed_frames_ GUARDED_BY(crit_); 181 182 // Holds the information about the last completed frame for a given temporal 183 // layer given a Tl0 picture index. 184 std::map<uint8_t, 185 std::array<int16_t, kMaxTemporalLayers>, 186 DescendingSeqNumComp<uint8_t>> 187 layer_info_ GUARDED_BY(crit_); 188 189 // Where the current scalability structure is in the 190 // |scalability_structures_| array. 191 uint8_t current_ss_idx_; 192 193 // Holds received scalability structures. 194 std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_ 195 GUARDED_BY(crit_); 196 197 // Holds the the Gof information for a given TL0 picture index. 198 std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_ 199 GUARDED_BY(crit_); 200 201 // Keep track of which picture id and which temporal layer that had the 202 // up switch flag set. 203 std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>> 204 up_switch_ GUARDED_BY(crit_); 205 206 // For every temporal layer, keep a set of which frames that are missing. 207 std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>, 208 kMaxTemporalLayers> 209 missing_frames_for_layer_ GUARDED_BY(crit_); 210 211 // How far frames have been cleared by sequence number. A frame will be 212 // cleared if it contains a packet with a sequence number older than 213 // |cleared_to_seq_num_|. 214 int cleared_to_seq_num_ GUARDED_BY(crit_); 215 216 OnCompleteFrameCallback* frame_callback_; 217 218 // Vp9PidFix variables 219 // TODO(philipel): Remove when VP9 PID does not jump mid-stream. 220 int vp9_fix_last_timestamp_ = -1; 221 int vp9_fix_jump_timestamp_ = -1; 222 int vp9_fix_last_picture_id_ = -1; 223 int vp9_fix_pid_offset_ = 0; 224 int vp9_fix_last_tl0_pic_idx_ = -1; 225 int vp9_fix_tl0_pic_idx_offset_ = 0; 226 }; 227 228 } // namespace video_coding 229 } // namespace webrtc 230 231 #endif // WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ 232