1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "video/video_quality_observer2.h"
12 
13 #include <algorithm>
14 #include <cmath>
15 #include <cstdint>
16 #include <string>
17 
18 #include "rtc_base/logging.h"
19 #include "rtc_base/strings/string_builder.h"
20 #include "system_wrappers/include/metrics.h"
21 #include "video/video_receive_stream2.h"
22 
23 namespace webrtc {
24 namespace internal {
25 const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
26 const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
27 const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;
28 
29 namespace {
30 constexpr int kMinVideoDurationMs = 3000;
31 constexpr int kMinRequiredSamples = 1;
32 constexpr int kPixelsInHighResolution =
33     960 * 540;  // CPU-adapted HD still counts.
34 constexpr int kPixelsInMediumResolution = 640 * 360;
35 constexpr int kBlockyQpThresholdVp8 = 70;
36 constexpr int kBlockyQpThresholdVp9 = 180;
37 constexpr int kMaxNumCachedBlockyFrames = 100;
38 // TODO(ilnik): Add H264/HEVC thresholds.
39 }  // namespace
40 
VideoQualityObserver()41 VideoQualityObserver::VideoQualityObserver()
42     : last_frame_rendered_ms_(-1),
43       num_frames_rendered_(0),
44       first_frame_rendered_ms_(-1),
45       last_frame_pixels_(0),
46       is_last_frame_blocky_(false),
47       last_unfreeze_time_ms_(0),
48       render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
49       sum_squared_interframe_delays_secs_(0.0),
50       time_in_resolution_ms_(3, 0),
51       current_resolution_(Resolution::Low),
52       num_resolution_downgrades_(0),
53       time_in_blocky_video_ms_(0),
54       is_paused_(false) {}
55 
UpdateHistograms(bool screenshare)56 void VideoQualityObserver::UpdateHistograms(bool screenshare) {
57   // TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_
58   // be the same as the construction thread.
59 
60   // Don't report anything on an empty video stream.
61   if (num_frames_rendered_ == 0) {
62     return;
63   }
64 
65   char log_stream_buf[2 * 1024];
66   rtc::SimpleStringBuilder log_stream(log_stream_buf);
67 
68   if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
69     smooth_playback_durations_.Add(last_frame_rendered_ms_ -
70                                    last_unfreeze_time_ms_);
71   }
72 
73   std::string uma_prefix =
74       screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video";
75 
76   auto mean_time_between_freezes =
77       smooth_playback_durations_.Avg(kMinRequiredSamples);
78   if (mean_time_between_freezes) {
79     RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
80                                        *mean_time_between_freezes);
81     log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
82                << *mean_time_between_freezes << "\n";
83   }
84   auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
85   if (avg_freeze_length) {
86     RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
87                                        *avg_freeze_length);
88     log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
89                << "\n";
90   }
91 
92   int64_t video_duration_ms =
93       last_frame_rendered_ms_ - first_frame_rendered_ms_;
94 
95   if (video_duration_ms >= kMinVideoDurationMs) {
96     int time_spent_in_hd_percentage = static_cast<int>(
97         time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
98     RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
99                                     time_spent_in_hd_percentage);
100     log_stream << uma_prefix << ".TimeInHdPercentage "
101                << time_spent_in_hd_percentage << "\n";
102 
103     int time_with_blocky_video_percentage =
104         static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
105     RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
106                                     time_with_blocky_video_percentage);
107     log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
108                << time_with_blocky_video_percentage << "\n";
109 
110     int num_resolution_downgrades_per_minute =
111         num_resolution_downgrades_ * 60000 / video_duration_ms;
112     RTC_HISTOGRAM_COUNTS_SPARSE_100(
113         uma_prefix + ".NumberResolutionDownswitchesPerMinute",
114         num_resolution_downgrades_per_minute);
115     log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
116                << num_resolution_downgrades_per_minute << "\n";
117 
118     int num_freezes_per_minute =
119         freezes_durations_.NumSamples() * 60000 / video_duration_ms;
120     RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
121                                     num_freezes_per_minute);
122     log_stream << uma_prefix << ".NumberFreezesPerMinute "
123                << num_freezes_per_minute << "\n";
124 
125     if (sum_squared_interframe_delays_secs_ > 0.0) {
126       int harmonic_framerate_fps = std::round(
127           video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
128       RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
129                                       harmonic_framerate_fps);
130       log_stream << uma_prefix << ".HarmonicFrameRate "
131                  << harmonic_framerate_fps << "\n";
132     }
133   }
134   RTC_LOG(LS_INFO) << log_stream.str();
135 }
136 
OnRenderedFrame(const VideoFrameMetaData & frame_meta)137 void VideoQualityObserver::OnRenderedFrame(
138     const VideoFrameMetaData& frame_meta) {
139   RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms());
140   RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms());
141 
142   if (num_frames_rendered_ == 0) {
143     first_frame_rendered_ms_ = last_unfreeze_time_ms_ =
144         frame_meta.decode_timestamp.ms();
145   }
146 
147   auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp);
148 
149   if (num_frames_rendered_ > 0) {
150     // Process inter-frame delay.
151     const int64_t interframe_delay_ms =
152         frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_;
153     const double interframe_delays_secs = interframe_delay_ms / 1000.0;
154 
155     // Sum of squared inter frame intervals is used to calculate the harmonic
156     // frame rate metric. The metric aims to reflect overall experience related
157     // to smoothness of video playback and includes both freezes and pauses.
158     sum_squared_interframe_delays_secs_ +=
159         interframe_delays_secs * interframe_delays_secs;
160 
161     if (!is_paused_) {
162       render_interframe_delays_.AddSample(interframe_delay_ms);
163 
164       bool was_freeze = false;
165       if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
166         const absl::optional<int64_t> avg_interframe_delay =
167             render_interframe_delays_.GetAverageRoundedDown();
168         RTC_DCHECK(avg_interframe_delay);
169         was_freeze = interframe_delay_ms >=
170                      std::max(3 * *avg_interframe_delay,
171                               *avg_interframe_delay + kMinIncreaseForFreezeMs);
172       }
173 
174       if (was_freeze) {
175         freezes_durations_.Add(interframe_delay_ms);
176         smooth_playback_durations_.Add(last_frame_rendered_ms_ -
177                                        last_unfreeze_time_ms_);
178         last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
179       } else {
180         // Count spatial metrics if there were no freeze.
181         time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
182 
183         if (is_last_frame_blocky_) {
184           time_in_blocky_video_ms_ += interframe_delay_ms;
185         }
186       }
187     }
188   }
189 
190   if (is_paused_) {
191     // If the stream was paused since the previous frame, do not count the
192     // pause toward smooth playback. Explicitly count the part before it and
193     // start the new smooth playback interval from this frame.
194     is_paused_ = false;
195     if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
196       smooth_playback_durations_.Add(last_frame_rendered_ms_ -
197                                      last_unfreeze_time_ms_);
198     }
199     last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms();
200 
201     if (num_frames_rendered_ > 0) {
202       pauses_durations_.Add(frame_meta.decode_timestamp.ms() -
203                             last_frame_rendered_ms_);
204     }
205   }
206 
207   int64_t pixels = frame_meta.width * frame_meta.height;
208   if (pixels >= kPixelsInHighResolution) {
209     current_resolution_ = Resolution::High;
210   } else if (pixels >= kPixelsInMediumResolution) {
211     current_resolution_ = Resolution::Medium;
212   } else {
213     current_resolution_ = Resolution::Low;
214   }
215 
216   if (pixels < last_frame_pixels_) {
217     ++num_resolution_downgrades_;
218   }
219 
220   last_frame_pixels_ = pixels;
221   last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms();
222 
223   is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
224   if (is_last_frame_blocky_) {
225     blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
226   }
227 
228   ++num_frames_rendered_;
229 }
230 
OnDecodedFrame(uint32_t rtp_frame_timestamp,absl::optional<uint8_t> qp,VideoCodecType codec)231 void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp,
232                                           absl::optional<uint8_t> qp,
233                                           VideoCodecType codec) {
234   if (!qp)
235     return;
236 
237   absl::optional<int> qp_blocky_threshold;
238   // TODO(ilnik): add other codec types when we have QP for them.
239   switch (codec) {
240     case kVideoCodecVP8:
241       qp_blocky_threshold = kBlockyQpThresholdVp8;
242       break;
243     case kVideoCodecVP9:
244       qp_blocky_threshold = kBlockyQpThresholdVp9;
245       break;
246     default:
247       qp_blocky_threshold = absl::nullopt;
248   }
249 
250   RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end());
251 
252   if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
253     // Cache blocky frame. Its duration will be calculated in render callback.
254     if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
255       RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
256       blocky_frames_.erase(
257           blocky_frames_.begin(),
258           std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
259     }
260 
261     blocky_frames_.insert(rtp_frame_timestamp);
262   }
263 }
264 
OnStreamInactive()265 void VideoQualityObserver::OnStreamInactive() {
266   is_paused_ = true;
267 }
268 
NumFreezes() const269 uint32_t VideoQualityObserver::NumFreezes() const {
270   return freezes_durations_.NumSamples();
271 }
272 
NumPauses() const273 uint32_t VideoQualityObserver::NumPauses() const {
274   return pauses_durations_.NumSamples();
275 }
276 
TotalFreezesDurationMs() const277 uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
278   return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
279 }
280 
TotalPausesDurationMs() const281 uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
282   return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
283 }
284 
TotalFramesDurationMs() const285 uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
286   return last_frame_rendered_ms_ - first_frame_rendered_ms_;
287 }
288 
SumSquaredFrameDurationsSec() const289 double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
290   return sum_squared_interframe_delays_secs_;
291 }
292 
293 }  // namespace internal
294 }  // namespace webrtc
295