1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef VIDEO_RECEIVE_STATISTICS_PROXY2_H_
12 #define VIDEO_RECEIVE_STATISTICS_PROXY2_H_
13 
14 #include <map>
15 #include <memory>
16 #include <string>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/task_queue/task_queue_base.h"
21 #include "api/units/timestamp.h"
22 #include "call/video_receive_stream.h"
23 #include "modules/include/module_common_types.h"
24 #include "modules/video_coding/include/video_coding_defines.h"
25 #include "rtc_base/numerics/histogram_percentile_counter.h"
26 #include "rtc_base/numerics/moving_max_counter.h"
27 #include "rtc_base/numerics/sample_counter.h"
28 #include "rtc_base/rate_statistics.h"
29 #include "rtc_base/rate_tracker.h"
30 #include "rtc_base/synchronization/sequence_checker.h"
31 #include "rtc_base/task_utils/pending_task_safety_flag.h"
32 #include "rtc_base/thread_annotations.h"
33 #include "rtc_base/thread_checker.h"
34 #include "video/quality_threshold.h"
35 #include "video/stats_counter.h"
36 #include "video/video_quality_observer2.h"
37 
38 namespace webrtc {
39 
40 class Clock;
41 struct CodecSpecificInfo;
42 
43 namespace internal {
44 // Declared in video_receive_stream2.h.
45 struct VideoFrameMetaData;
46 
47 class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
48                                public RtcpCnameCallback,
49                                public RtcpPacketTypeCounterObserver {
50  public:
51   ReceiveStatisticsProxy(const VideoReceiveStream::Config* config,
52                          Clock* clock,
53                          TaskQueueBase* worker_thread);
54   ~ReceiveStatisticsProxy() override;
55 
56   VideoReceiveStream::Stats GetStats() const;
57 
58   void OnDecodedFrame(const VideoFrame& frame,
59                       absl::optional<uint8_t> qp,
60                       int32_t decode_time_ms,
61                       VideoContentType content_type);
62 
63   // Called asyncronously on the worker thread as a result of a call to the
64   // above OnDecodedFrame method, which is called back on the thread where
65   // the actual decoding happens.
66   void OnDecodedFrame(const VideoFrameMetaData& frame_meta,
67                       absl::optional<uint8_t> qp,
68                       int32_t decode_time_ms,
69                       VideoContentType content_type);
70 
71   void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms,
72                            int64_t sync_offset_ms,
73                            double estimated_freq_khz);
74   void OnRenderedFrame(const VideoFrameMetaData& frame_meta);
75   void OnIncomingPayloadType(int payload_type);
76   void OnDecoderImplementationName(const char* implementation_name);
77 
78   void OnPreDecode(VideoCodecType codec_type, int qp);
79 
80   void OnUniqueFramesCounted(int num_unique_frames);
81 
82   // Indicates video stream has been paused (no incoming packets).
83   void OnStreamInactive();
84 
85   // Overrides VCMReceiveStatisticsCallback.
86   void OnCompleteFrame(bool is_keyframe,
87                        size_t size_bytes,
88                        VideoContentType content_type) override;
89   void OnDroppedFrames(uint32_t frames_dropped) override;
90   void OnDiscardedPackets(uint32_t packets_discarded) override;
91   void OnFrameBufferTimingsUpdated(int max_decode_ms,
92                                    int current_delay_ms,
93                                    int target_delay_ms,
94                                    int jitter_buffer_ms,
95                                    int min_playout_delay_ms,
96                                    int render_delay_ms) override;
97 
98   void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override;
99 
100   // Overrides RtcpCnameCallback.
101   void OnCname(uint32_t ssrc, absl::string_view cname) override;
102 
103   // Overrides RtcpPacketTypeCounterObserver.
104   void RtcpPacketTypesCounterUpdated(
105       uint32_t ssrc,
106       const RtcpPacketTypeCounter& packet_counter) override;
107 
108   void OnRttUpdate(int64_t avg_rtt_ms);
109 
110   // Notification methods that are used to check our internal state and validate
111   // threading assumptions. These are called by VideoReceiveStream.
112   void DecoderThreadStarting();
113   void DecoderThreadStopped();
114 
115   // Produce histograms. Must be called after DecoderThreadStopped(), typically
116   // at the end of the call.
117   void UpdateHistograms(absl::optional<int> fraction_lost,
118                         const StreamDataCounters& rtp_stats,
119                         const StreamDataCounters* rtx_stats);
120 
121  private:
122   struct QpCounters {
123     rtc::SampleCounter vp8;
124   };
125 
126   struct ContentSpecificStats {
127     ContentSpecificStats();
128     ~ContentSpecificStats();
129 
130     void Add(const ContentSpecificStats& other);
131 
132     rtc::SampleCounter e2e_delay_counter;
133     rtc::SampleCounter interframe_delay_counter;
134     int64_t flow_duration_ms = 0;
135     int64_t total_media_bytes = 0;
136     rtc::SampleCounter received_width;
137     rtc::SampleCounter received_height;
138     rtc::SampleCounter qp_counter;
139     FrameCounts frame_counts;
140     rtc::HistogramPercentileCounter interframe_delay_percentiles;
141   };
142 
143   void QualitySample(Timestamp now);
144 
145   // Removes info about old frames and then updates the framerate.
146   void UpdateFramerate(int64_t now_ms) const;
147 
148   void UpdateDecodeTimeHistograms(int width,
149                                   int height,
150                                   int decode_time_ms) const;
151 
152   absl::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs(
153       int64_t now_ms) const;
154 
155   Clock* const clock_;
156   const int64_t start_ms_;
157   const bool enable_decode_time_histograms_;
158 
159   int64_t last_sample_time_ RTC_GUARDED_BY(main_thread_);
160 
161   QualityThreshold fps_threshold_ RTC_GUARDED_BY(main_thread_);
162   QualityThreshold qp_threshold_ RTC_GUARDED_BY(main_thread_);
163   QualityThreshold variance_threshold_ RTC_GUARDED_BY(main_thread_);
164   rtc::SampleCounter qp_sample_ RTC_GUARDED_BY(main_thread_);
165   int num_bad_states_ RTC_GUARDED_BY(main_thread_);
166   int num_certain_states_ RTC_GUARDED_BY(main_thread_);
167   // Note: The |stats_.rtp_stats| member is not used or populated by this class.
168   mutable VideoReceiveStream::Stats stats_ RTC_GUARDED_BY(main_thread_);
169   // Same as stats_.ssrc, but const (no lock required).
170   const uint32_t remote_ssrc_;
171   RateStatistics decode_fps_estimator_ RTC_GUARDED_BY(main_thread_);
172   RateStatistics renders_fps_estimator_ RTC_GUARDED_BY(main_thread_);
173   rtc::RateTracker render_fps_tracker_ RTC_GUARDED_BY(main_thread_);
174   rtc::RateTracker render_pixel_tracker_ RTC_GUARDED_BY(main_thread_);
175   rtc::SampleCounter sync_offset_counter_ RTC_GUARDED_BY(main_thread_);
176   rtc::SampleCounter decode_time_counter_ RTC_GUARDED_BY(main_thread_);
177   rtc::SampleCounter jitter_buffer_delay_counter_ RTC_GUARDED_BY(main_thread_);
178   rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(main_thread_);
179   rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(main_thread_);
180   rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(main_thread_);
181   std::unique_ptr<VideoQualityObserver> video_quality_observer_
182       RTC_GUARDED_BY(main_thread_);
183   mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_
184       RTC_GUARDED_BY(main_thread_);
185   std::map<VideoContentType, ContentSpecificStats> content_specific_stats_
186       RTC_GUARDED_BY(main_thread_);
187   MaxCounter freq_offset_counter_ RTC_GUARDED_BY(main_thread_);
188   QpCounters qp_counters_ RTC_GUARDED_BY(main_thread_);
189   int64_t avg_rtt_ms_ RTC_GUARDED_BY(main_thread_) = 0;
190   mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(main_thread_);
191   VideoContentType last_content_type_ RTC_GUARDED_BY(&main_thread_);
192   VideoCodecType last_codec_type_ RTC_GUARDED_BY(main_thread_);
193   absl::optional<int64_t> first_frame_received_time_ms_
194       RTC_GUARDED_BY(main_thread_);
195   absl::optional<int64_t> first_decoded_frame_time_ms_
196       RTC_GUARDED_BY(main_thread_);
197   absl::optional<int64_t> last_decoded_frame_time_ms_
198       RTC_GUARDED_BY(main_thread_);
199   size_t num_delayed_frames_rendered_ RTC_GUARDED_BY(main_thread_);
200   int64_t sum_missed_render_deadline_ms_ RTC_GUARDED_BY(main_thread_);
201   // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is
202   // called from const GetStats().
203   mutable rtc::MovingMaxCounter<TimingFrameInfo> timing_frame_info_counter_
204       RTC_GUARDED_BY(main_thread_);
205   absl::optional<int> num_unique_frames_ RTC_GUARDED_BY(main_thread_);
206   absl::optional<int64_t> last_estimated_playout_ntp_timestamp_ms_
207       RTC_GUARDED_BY(main_thread_);
208   absl::optional<int64_t> last_estimated_playout_time_ms_
209       RTC_GUARDED_BY(main_thread_);
210 
211   // The thread on which this instance is constructed and some of its main
212   // methods are invoked on such as GetStats().
213   TaskQueueBase* const worker_thread_;
214 
215   ScopedTaskSafety task_safety_;
216 
217   SequenceChecker decode_queue_;
218   rtc::ThreadChecker main_thread_;
219   SequenceChecker incoming_render_queue_;
220 };
221 
222 }  // namespace internal
223 }  // namespace webrtc
224 #endif  // VIDEO_RECEIVE_STATISTICS_PROXY2_H_
225