1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef MODULES_VIDEO_CODING_FRAME_BUFFER2_H_
12 #define MODULES_VIDEO_CODING_FRAME_BUFFER2_H_
13 
14 #include <array>
15 #include <map>
16 #include <memory>
17 #include <utility>
18 
19 #include "modules/video_coding/frame_object.h"
20 #include "modules/video_coding/include/video_coding_defines.h"
21 #include "modules/video_coding/inter_frame_delay.h"
22 #include "rtc_base/constructormagic.h"
23 #include "rtc_base/criticalsection.h"
24 #include "rtc_base/event.h"
25 #include "rtc_base/numerics/sequence_number_util.h"
26 #include "rtc_base/thread_annotations.h"
27 
28 namespace webrtc {
29 
30 class Clock;
31 class VCMReceiveStatisticsCallback;
32 class VCMJitterEstimator;
33 class VCMTiming;
34 
35 namespace video_coding {
36 
37 class FrameBuffer {
38  public:
39   enum ReturnReason { kFrameFound, kTimeout, kStopped };
40 
41   FrameBuffer(Clock* clock,
42               VCMJitterEstimator* jitter_estimator,
43               VCMTiming* timing,
44               VCMReceiveStatisticsCallback* stats_proxy);
45 
46   virtual ~FrameBuffer();
47 
48   // Insert a frame into the frame buffer. Returns the picture id
49   // of the last continuous frame or -1 if there is no continuous frame.
50   int InsertFrame(std::unique_ptr<FrameObject> frame);
51 
52   // Get the next frame for decoding. Will return at latest after
53   // |max_wait_time_ms|.
54   //  - If a frame is available within |max_wait_time_ms| it will return
55   //    kFrameFound and set |frame_out| to the resulting frame.
56   //  - If no frame is available after |max_wait_time_ms| it will return
57   //    kTimeout.
58   //  - If the FrameBuffer is stopped then it will return kStopped.
59   ReturnReason NextFrame(int64_t max_wait_time_ms,
60                          std::unique_ptr<FrameObject>* frame_out,
61                          bool keyframe_required = false);
62 
63   // Tells the FrameBuffer which protection mode that is in use. Affects
64   // the frame timing.
65   // TODO(philipel): Remove this when new timing calculations has been
66   //                 implemented.
67   void SetProtectionMode(VCMVideoProtection mode);
68 
69   // Start the frame buffer, has no effect if the frame buffer is started.
70   // The frame buffer is started upon construction.
71   void Start();
72 
73   // Stop the frame buffer, causing any sleeping thread in NextFrame to
74   // return immediately.
75   void Stop();
76 
77   // Updates the RTT for jitter buffer estimation.
78   void UpdateRtt(int64_t rtt_ms);
79 
80  private:
81   struct FrameKey {
FrameKeyFrameKey82     FrameKey() : picture_id(-1), spatial_layer(0) {}
FrameKeyFrameKey83     FrameKey(int64_t picture_id, uint8_t spatial_layer)
84         : picture_id(picture_id), spatial_layer(spatial_layer) {}
85 
86     bool operator<(const FrameKey& rhs) const {
87       if (picture_id == rhs.picture_id)
88         return spatial_layer < rhs.spatial_layer;
89       return picture_id < rhs.picture_id;
90     }
91 
92     bool operator<=(const FrameKey& rhs) const { return !(rhs < *this); }
93 
94     int64_t picture_id;
95     uint8_t spatial_layer;
96   };
97 
98   struct FrameInfo {
99     // The maximum number of frames that can depend on this frame.
100     static constexpr size_t kMaxNumDependentFrames = 8;
101 
102     // Which other frames that have direct unfulfilled dependencies
103     // on this frame.
104     // TODO(philipel): Add simple modify/access functions to prevent adding too
105     // many |dependent_frames|.
106     FrameKey dependent_frames[kMaxNumDependentFrames];
107     size_t num_dependent_frames = 0;
108 
109     // A frame is continiuous if it has all its referenced/indirectly
110     // referenced frames.
111     //
112     // How many unfulfilled frames this frame have until it becomes continuous.
113     size_t num_missing_continuous = 0;
114 
115     // A frame is decodable if all its referenced frames have been decoded.
116     //
117     // How many unfulfilled frames this frame have until it becomes decodable.
118     size_t num_missing_decodable = 0;
119 
120     // If this frame is continuous or not.
121     bool continuous = false;
122 
123     // The actual FrameObject.
124     std::unique_ptr<FrameObject> frame;
125   };
126 
127   using FrameMap = std::map<FrameKey, FrameInfo>;
128 
129   // Check that the references of |frame| are valid.
130   bool ValidReferences(const FrameObject& frame) const;
131 
132   // Updates the minimal and maximal playout delays
133   // depending on the frame.
134   void UpdatePlayoutDelays(const FrameObject& frame)
135       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
136 
137   // Update all directly dependent and indirectly dependent frames and mark
138   // them as continuous if all their references has been fulfilled.
139   void PropagateContinuity(FrameMap::iterator start)
140       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
141 
142   // Marks the frame as decoded and updates all directly dependent frames.
143   void PropagateDecodability(const FrameInfo& info)
144       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
145 
146   // Advances |last_decoded_frame_it_| to |decoded| and removes old
147   // frame info.
148   void AdvanceLastDecodedFrame(FrameMap::iterator decoded)
149       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
150 
151   // Update the corresponding FrameInfo of |frame| and all FrameInfos that
152   // |frame| references.
153   // Return false if |frame| will never be decodable, true otherwise.
154   bool UpdateFrameInfoWithIncomingFrame(const FrameObject& frame,
155                                         FrameMap::iterator info)
156       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
157 
158   void UpdateJitterDelay() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
159 
160   void UpdateTimingFrameInfo() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
161 
162   void ClearFramesAndHistory() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
163 
164   bool HasBadRenderTiming(const FrameObject& frame, int64_t now_ms)
165       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
166 
167   FrameMap frames_ RTC_GUARDED_BY(crit_);
168 
169   rtc::CriticalSection crit_;
170   Clock* const clock_;
171   rtc::Event new_continuous_frame_event_;
172   VCMJitterEstimator* const jitter_estimator_ RTC_GUARDED_BY(crit_);
173   VCMTiming* const timing_ RTC_GUARDED_BY(crit_);
174   VCMInterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(crit_);
175   uint32_t last_decoded_frame_timestamp_ RTC_GUARDED_BY(crit_);
176   FrameMap::iterator last_decoded_frame_it_ RTC_GUARDED_BY(crit_);
177   FrameMap::iterator last_continuous_frame_it_ RTC_GUARDED_BY(crit_);
178   FrameMap::iterator next_frame_it_ RTC_GUARDED_BY(crit_);
179   int num_frames_history_ RTC_GUARDED_BY(crit_);
180   int num_frames_buffered_ RTC_GUARDED_BY(crit_);
181   bool stopped_ RTC_GUARDED_BY(crit_);
182   VCMVideoProtection protection_mode_ RTC_GUARDED_BY(crit_);
183   VCMReceiveStatisticsCallback* const stats_callback_;
184   int64_t last_log_non_decoded_ms_ RTC_GUARDED_BY(crit_);
185 
186   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer);
187 };
188 
189 }  // namespace video_coding
190 }  // namespace webrtc
191 
192 #endif  // MODULES_VIDEO_CODING_FRAME_BUFFER2_H_
193