1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 12 #define VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/task_queue/task_queue_base.h" 21 #include "api/units/timestamp.h" 22 #include "call/video_receive_stream.h" 23 #include "modules/include/module_common_types.h" 24 #include "modules/video_coding/include/video_coding_defines.h" 25 #include "rtc_base/numerics/histogram_percentile_counter.h" 26 #include "rtc_base/numerics/moving_max_counter.h" 27 #include "rtc_base/numerics/sample_counter.h" 28 #include "rtc_base/rate_statistics.h" 29 #include "rtc_base/rate_tracker.h" 30 #include "rtc_base/synchronization/sequence_checker.h" 31 #include "rtc_base/task_utils/pending_task_safety_flag.h" 32 #include "rtc_base/thread_annotations.h" 33 #include "rtc_base/thread_checker.h" 34 #include "video/quality_threshold.h" 35 #include "video/stats_counter.h" 36 #include "video/video_quality_observer2.h" 37 38 namespace webrtc { 39 40 class Clock; 41 struct CodecSpecificInfo; 42 43 namespace internal { 44 // Declared in video_receive_stream2.h. 45 struct VideoFrameMetaData; 46 47 class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, 48 public RtcpCnameCallback, 49 public RtcpPacketTypeCounterObserver { 50 public: 51 ReceiveStatisticsProxy(const VideoReceiveStream::Config* config, 52 Clock* clock, 53 TaskQueueBase* worker_thread); 54 ~ReceiveStatisticsProxy() override; 55 56 VideoReceiveStream::Stats GetStats() const; 57 58 void OnDecodedFrame(const VideoFrame& frame, 59 absl::optional<uint8_t> qp, 60 int32_t decode_time_ms, 61 VideoContentType content_type); 62 63 // Called asyncronously on the worker thread as a result of a call to the 64 // above OnDecodedFrame method, which is called back on the thread where 65 // the actual decoding happens. 66 void OnDecodedFrame(const VideoFrameMetaData& frame_meta, 67 absl::optional<uint8_t> qp, 68 int32_t decode_time_ms, 69 VideoContentType content_type); 70 71 void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, 72 int64_t sync_offset_ms, 73 double estimated_freq_khz); 74 void OnRenderedFrame(const VideoFrameMetaData& frame_meta); 75 void OnIncomingPayloadType(int payload_type); 76 void OnDecoderImplementationName(const char* implementation_name); 77 78 void OnPreDecode(VideoCodecType codec_type, int qp); 79 80 void OnUniqueFramesCounted(int num_unique_frames); 81 82 // Indicates video stream has been paused (no incoming packets). 83 void OnStreamInactive(); 84 85 // Overrides VCMReceiveStatisticsCallback. 86 void OnCompleteFrame(bool is_keyframe, 87 size_t size_bytes, 88 VideoContentType content_type) override; 89 void OnDroppedFrames(uint32_t frames_dropped) override; 90 void OnDiscardedPackets(uint32_t packets_discarded) override; 91 void OnFrameBufferTimingsUpdated(int max_decode_ms, 92 int current_delay_ms, 93 int target_delay_ms, 94 int jitter_buffer_ms, 95 int min_playout_delay_ms, 96 int render_delay_ms) override; 97 98 void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) override; 99 100 // Overrides RtcpCnameCallback. 101 void OnCname(uint32_t ssrc, absl::string_view cname) override; 102 103 // Overrides RtcpPacketTypeCounterObserver. 104 void RtcpPacketTypesCounterUpdated( 105 uint32_t ssrc, 106 const RtcpPacketTypeCounter& packet_counter) override; 107 108 void OnRttUpdate(int64_t avg_rtt_ms); 109 110 // Notification methods that are used to check our internal state and validate 111 // threading assumptions. These are called by VideoReceiveStream. 112 void DecoderThreadStarting(); 113 void DecoderThreadStopped(); 114 115 // Produce histograms. Must be called after DecoderThreadStopped(), typically 116 // at the end of the call. 117 void UpdateHistograms(absl::optional<int> fraction_lost, 118 const StreamDataCounters& rtp_stats, 119 const StreamDataCounters* rtx_stats); 120 121 private: 122 struct QpCounters { 123 rtc::SampleCounter vp8; 124 }; 125 126 struct ContentSpecificStats { 127 ContentSpecificStats(); 128 ~ContentSpecificStats(); 129 130 void Add(const ContentSpecificStats& other); 131 132 rtc::SampleCounter e2e_delay_counter; 133 rtc::SampleCounter interframe_delay_counter; 134 int64_t flow_duration_ms = 0; 135 int64_t total_media_bytes = 0; 136 rtc::SampleCounter received_width; 137 rtc::SampleCounter received_height; 138 rtc::SampleCounter qp_counter; 139 FrameCounts frame_counts; 140 rtc::HistogramPercentileCounter interframe_delay_percentiles; 141 }; 142 143 void QualitySample(Timestamp now); 144 145 // Removes info about old frames and then updates the framerate. 146 void UpdateFramerate(int64_t now_ms) const; 147 148 void UpdateDecodeTimeHistograms(int width, 149 int height, 150 int decode_time_ms) const; 151 152 absl::optional<int64_t> GetCurrentEstimatedPlayoutNtpTimestampMs( 153 int64_t now_ms) const; 154 155 Clock* const clock_; 156 const int64_t start_ms_; 157 const bool enable_decode_time_histograms_; 158 159 int64_t last_sample_time_ RTC_GUARDED_BY(main_thread_); 160 161 QualityThreshold fps_threshold_ RTC_GUARDED_BY(main_thread_); 162 QualityThreshold qp_threshold_ RTC_GUARDED_BY(main_thread_); 163 QualityThreshold variance_threshold_ RTC_GUARDED_BY(main_thread_); 164 rtc::SampleCounter qp_sample_ RTC_GUARDED_BY(main_thread_); 165 int num_bad_states_ RTC_GUARDED_BY(main_thread_); 166 int num_certain_states_ RTC_GUARDED_BY(main_thread_); 167 // Note: The |stats_.rtp_stats| member is not used or populated by this class. 168 mutable VideoReceiveStream::Stats stats_ RTC_GUARDED_BY(main_thread_); 169 // Same as stats_.ssrc, but const (no lock required). 170 const uint32_t remote_ssrc_; 171 RateStatistics decode_fps_estimator_ RTC_GUARDED_BY(main_thread_); 172 RateStatistics renders_fps_estimator_ RTC_GUARDED_BY(main_thread_); 173 rtc::RateTracker render_fps_tracker_ RTC_GUARDED_BY(main_thread_); 174 rtc::RateTracker render_pixel_tracker_ RTC_GUARDED_BY(main_thread_); 175 rtc::SampleCounter sync_offset_counter_ RTC_GUARDED_BY(main_thread_); 176 rtc::SampleCounter decode_time_counter_ RTC_GUARDED_BY(main_thread_); 177 rtc::SampleCounter jitter_buffer_delay_counter_ RTC_GUARDED_BY(main_thread_); 178 rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(main_thread_); 179 rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(main_thread_); 180 rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(main_thread_); 181 std::unique_ptr<VideoQualityObserver> video_quality_observer_ 182 RTC_GUARDED_BY(main_thread_); 183 mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_ 184 RTC_GUARDED_BY(main_thread_); 185 std::map<VideoContentType, ContentSpecificStats> content_specific_stats_ 186 RTC_GUARDED_BY(main_thread_); 187 MaxCounter freq_offset_counter_ RTC_GUARDED_BY(main_thread_); 188 QpCounters qp_counters_ RTC_GUARDED_BY(main_thread_); 189 int64_t avg_rtt_ms_ RTC_GUARDED_BY(main_thread_) = 0; 190 mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(main_thread_); 191 VideoContentType last_content_type_ RTC_GUARDED_BY(&main_thread_); 192 VideoCodecType last_codec_type_ RTC_GUARDED_BY(main_thread_); 193 absl::optional<int64_t> first_frame_received_time_ms_ 194 RTC_GUARDED_BY(main_thread_); 195 absl::optional<int64_t> first_decoded_frame_time_ms_ 196 RTC_GUARDED_BY(main_thread_); 197 absl::optional<int64_t> last_decoded_frame_time_ms_ 198 RTC_GUARDED_BY(main_thread_); 199 size_t num_delayed_frames_rendered_ RTC_GUARDED_BY(main_thread_); 200 int64_t sum_missed_render_deadline_ms_ RTC_GUARDED_BY(main_thread_); 201 // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is 202 // called from const GetStats(). 203 mutable rtc::MovingMaxCounter<TimingFrameInfo> timing_frame_info_counter_ 204 RTC_GUARDED_BY(main_thread_); 205 absl::optional<int> num_unique_frames_ RTC_GUARDED_BY(main_thread_); 206 absl::optional<int64_t> last_estimated_playout_ntp_timestamp_ms_ 207 RTC_GUARDED_BY(main_thread_); 208 absl::optional<int64_t> last_estimated_playout_time_ms_ 209 RTC_GUARDED_BY(main_thread_); 210 211 // The thread on which this instance is constructed and some of its main 212 // methods are invoked on such as GetStats(). 213 TaskQueueBase* const worker_thread_; 214 215 ScopedTaskSafety task_safety_; 216 217 SequenceChecker decode_queue_; 218 rtc::ThreadChecker main_thread_; 219 SequenceChecker incoming_render_queue_; 220 }; 221 222 } // namespace internal 223 } // namespace webrtc 224 #endif // VIDEO_RECEIVE_STATISTICS_PROXY2_H_ 225