1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
12 #define WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
13 
14 #include <array>
15 #include <map>
16 #include <memory>
17 #include <deque>
18 #include <set>
19 #include <utility>
20 
21 #include "webrtc/base/criticalsection.h"
22 #include "webrtc/base/thread_annotations.h"
23 #include "webrtc/modules/include/module_common_types.h"
24 #include "webrtc/modules/video_coding/sequence_number_util.h"
25 
26 namespace webrtc {
27 namespace video_coding {
28 
29 class FrameObject;
30 class RtpFrameObject;
31 
32 // A complete frame is a frame which has received all its packets and all its
33 // references are known.
34 class OnCompleteFrameCallback {
35  public:
~OnCompleteFrameCallback()36   virtual ~OnCompleteFrameCallback() {}
37   virtual void OnCompleteFrame(std::unique_ptr<FrameObject> frame) = 0;
38 };
39 
40 class RtpFrameReferenceFinder {
41  public:
42   explicit RtpFrameReferenceFinder(OnCompleteFrameCallback* frame_callback);
43 
44   // Manage this frame until:
45   //  - We have all information needed to determine its references, after
46   //    which |frame_callback_| is called with the completed frame, or
47   //  - We have too many stashed frames (determined by |kMaxStashedFrames)
48   //    so we drop this frame, or
49   //  - It gets cleared by ClearTo, which also means we drop it.
50   void ManageFrame(std::unique_ptr<RtpFrameObject> frame);
51 
52   // Notifies that padding has been received, which the reference finder
53   // might need to calculate the references of a frame.
54   void PaddingReceived(uint16_t seq_num);
55 
56   // Clear all stashed frames that include packets older than |seq_num|.
57   void ClearTo(uint16_t seq_num);
58 
59  private:
60   static const uint16_t kPicIdLength = 1 << 15;
61   static const uint8_t kMaxTemporalLayers = 5;
62   static const int kMaxLayerInfo = 50;
63   static const int kMaxStashedFrames = 50;
64   static const int kMaxNotYetReceivedFrames = 100;
65   static const int kMaxGofSaved = 50;
66   static const int kMaxPaddingAge = 100;
67 
68 
69   struct GofInfo {
GofInfoGofInfo70     GofInfo(GofInfoVP9* gof, uint16_t last_picture_id)
71         : gof(gof), last_picture_id(last_picture_id) {}
72     GofInfoVP9* gof;
73     uint16_t last_picture_id;
74   };
75 
76   rtc::CriticalSection crit_;
77 
78   // Find the relevant group of pictures and update its "last-picture-id-with
79   // padding" sequence number.
80   void UpdateLastPictureIdWithPadding(uint16_t seq_num)
81       EXCLUSIVE_LOCKS_REQUIRED(crit_);
82 
83   // Retry finding references for all frames that previously didn't have
84   // all information needed.
85   void RetryStashedFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_);
86 
87   // Find references for generic frames. If |picture_id| is unspecified
88   // then packet sequence numbers will be used to determine the references
89   // of the frames.
90   void ManageFrameGeneric(std::unique_ptr<RtpFrameObject> frame,
91                           int picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_);
92 
93   // Find references for Vp8 frames
94   void ManageFrameVp8(std::unique_ptr<RtpFrameObject> frame)
95       EXCLUSIVE_LOCKS_REQUIRED(crit_);
96 
97   // Updates all necessary state used to determine frame references
98   // for Vp8 and then calls the |frame_callback| callback with the
99   // completed frame.
100   void CompletedFrameVp8(std::unique_ptr<RtpFrameObject> frame)
101       EXCLUSIVE_LOCKS_REQUIRED(crit_);
102 
103   // Find references for Vp9 frames
104   void ManageFrameVp9(std::unique_ptr<RtpFrameObject> frame)
105       EXCLUSIVE_LOCKS_REQUIRED(crit_);
106 
107   // Unwrap the picture id and the frame references  and then call the
108   // |frame_callback| callback with the completed frame.
109   void CompletedFrameVp9(std::unique_ptr<RtpFrameObject> frame)
110       EXCLUSIVE_LOCKS_REQUIRED(crit_);
111 
112   // Check if we are missing a frame necessary to determine the references
113   // for this frame.
114   bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info)
115       EXCLUSIVE_LOCKS_REQUIRED(crit_);
116 
117   // Updates which frames that have been received. If there is a gap,
118   // missing frames will be added to |missing_frames_for_layer_| or
119   // if this is an already missing frame then it will be removed.
120   void FrameReceivedVp9(uint16_t picture_id, GofInfo* info)
121       EXCLUSIVE_LOCKS_REQUIRED(crit_);
122 
123   // Check if there is a frame with the up-switch flag set in the interval
124   // (|pid_ref|, |picture_id|) with temporal layer smaller than |temporal_idx|.
125   bool UpSwitchInIntervalVp9(uint16_t picture_id,
126                              uint8_t temporal_idx,
127                              uint16_t pid_ref) EXCLUSIVE_LOCKS_REQUIRED(crit_);
128 
129   // All picture ids are unwrapped to 16 bits.
130   uint16_t UnwrapPictureId(uint16_t picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_);
131 
132   // Returns true if the frame is old and should be dropped.
133   // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
134   //                 around M59).
135   bool Vp9PidTl0Fix(const RtpFrameObject& frame,
136                     int16_t* picture_id,
137                     int16_t* tl0_pic_idx) EXCLUSIVE_LOCKS_REQUIRED(crit_);
138 
139   // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
140   //                 around M59).
141   bool DetectVp9PicIdJump(int fixed_pid,
142                           int fixed_tl0,
143                           uint32_t timestamp) const
144       EXCLUSIVE_LOCKS_REQUIRED(crit_);
145 
146   // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
147   //                 around M59).
148   bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const
149       EXCLUSIVE_LOCKS_REQUIRED(crit_);
150 
151   // For every group of pictures, hold two sequence numbers. The first being
152   // the sequence number of the last packet of the last completed frame, and
153   // the second being the sequence number of the last packet of the last
154   // completed frame advanced by any potential continuous packets of padding.
155   std::map<uint16_t,
156            std::pair<uint16_t, uint16_t>,
157            DescendingSeqNumComp<uint16_t>>
158       last_seq_num_gop_ GUARDED_BY(crit_);
159 
160   // Save the last picture id in order to detect when there is a gap in frames
161   // that have not yet been fully received.
162   int last_picture_id_ GUARDED_BY(crit_);
163 
164   // Padding packets that have been received but that are not yet continuous
165   // with any group of pictures.
166   std::set<uint16_t, DescendingSeqNumComp<uint16_t>> stashed_padding_
167       GUARDED_BY(crit_);
168 
169   // The last unwrapped picture id. Used to unwrap the picture id from a length
170   // of |kPicIdLength| to 16 bits.
171   int last_unwrap_ GUARDED_BY(crit_);
172 
173   // Frames earlier than the last received frame that have not yet been
174   // fully received.
175   std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
176       not_yet_received_frames_ GUARDED_BY(crit_);
177 
178   // Frames that have been fully received but didn't have all the information
179   // needed to determine their references.
180   std::deque<std::unique_ptr<RtpFrameObject>> stashed_frames_ GUARDED_BY(crit_);
181 
182   // Holds the information about the last completed frame for a given temporal
183   // layer given a Tl0 picture index.
184   std::map<uint8_t,
185            std::array<int16_t, kMaxTemporalLayers>,
186            DescendingSeqNumComp<uint8_t>>
187       layer_info_ GUARDED_BY(crit_);
188 
189   // Where the current scalability structure is in the
190   // |scalability_structures_| array.
191   uint8_t current_ss_idx_;
192 
193   // Holds received scalability structures.
194   std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_
195       GUARDED_BY(crit_);
196 
197   // Holds the the Gof information for a given TL0 picture index.
198   std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_
199       GUARDED_BY(crit_);
200 
201   // Keep track of which picture id and which temporal layer that had the
202   // up switch flag set.
203   std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
204       up_switch_ GUARDED_BY(crit_);
205 
206   // For every temporal layer, keep a set of which frames that are missing.
207   std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>,
208              kMaxTemporalLayers>
209       missing_frames_for_layer_ GUARDED_BY(crit_);
210 
211   // How far frames have been cleared by sequence number. A frame will be
212   // cleared if it contains a packet with a sequence number older than
213   // |cleared_to_seq_num_|.
214   int cleared_to_seq_num_ GUARDED_BY(crit_);
215 
216   OnCompleteFrameCallback* frame_callback_;
217 
218   // Vp9PidFix variables
219   // TODO(philipel): Remove when VP9 PID does not jump mid-stream.
220   int vp9_fix_last_timestamp_ = -1;
221   int vp9_fix_jump_timestamp_ = -1;
222   int vp9_fix_last_picture_id_ = -1;
223   int vp9_fix_pid_offset_ = 0;
224   int vp9_fix_last_tl0_pic_idx_ = -1;
225   int vp9_fix_tl0_pic_idx_offset_ = 0;
226 };
227 
228 }  // namespace video_coding
229 }  // namespace webrtc
230 
231 #endif  // WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
232