1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/utility/quality_scaler.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "rtc_base/checks.h"
17 #include "rtc_base/experiments/quality_scaler_settings.h"
18 #include "rtc_base/logging.h"
19 #include "rtc_base/numerics/exp_filter.h"
20 #include "rtc_base/task_queue.h"
21
22 // TODO(kthelgason): Some versions of Android have issues with log2.
23 // See https://code.google.com/p/android/issues/detail?id=212634 for details
24 #if defined(WEBRTC_ANDROID)
25 #define log2(x) (log(x) / log(2))
26 #endif
27
28 namespace webrtc {
29
30 namespace {
31 // TODO(nisse): Delete, delegate to encoders.
32 // Threshold constant used until first downscale (to permit fast rampup).
33 static const int kMeasureMs = 2000;
34 static const float kSamplePeriodScaleFactor = 2.5;
35 static const int kFramedropPercentThreshold = 60;
36 static const size_t kMinFramesNeededToScale = 2 * 30;
37
38 } // namespace
39
40 class QualityScaler::QpSmoother {
41 public:
QpSmoother(float alpha)42 explicit QpSmoother(float alpha)
43 : alpha_(alpha),
44 // The initial value of last_sample_ms doesn't matter since the smoother
45 // will ignore the time delta for the first update.
46 last_sample_ms_(0),
47 smoother_(alpha) {}
48
GetAvg() const49 absl::optional<int> GetAvg() const {
50 float value = smoother_.filtered();
51 if (value == rtc::ExpFilter::kValueUndefined) {
52 return absl::nullopt;
53 }
54 return static_cast<int>(value);
55 }
56
Add(float sample,int64_t time_sent_us)57 void Add(float sample, int64_t time_sent_us) {
58 int64_t now_ms = time_sent_us / 1000;
59 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
60 last_sample_ms_ = now_ms;
61 }
62
Reset()63 void Reset() { smoother_.Reset(alpha_); }
64
65 private:
66 const float alpha_;
67 int64_t last_sample_ms_;
68 rtc::ExpFilter smoother_;
69 };
70
QualityScaler(AdaptationObserverInterface * observer,VideoEncoder::QpThresholds thresholds)71 QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
72 VideoEncoder::QpThresholds thresholds)
73 : QualityScaler(observer, thresholds, kMeasureMs) {}
74
75 // Protected ctor, should not be called directly.
QualityScaler(AdaptationObserverInterface * observer,VideoEncoder::QpThresholds thresholds,int64_t sampling_period_ms)76 QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
77 VideoEncoder::QpThresholds thresholds,
78 int64_t sampling_period_ms)
79 : observer_(observer),
80 thresholds_(thresholds),
81 sampling_period_ms_(sampling_period_ms),
82 fast_rampup_(true),
83 // Arbitrarily choose size based on 30 fps for 5 seconds.
84 average_qp_(5 * 30),
85 framedrop_percent_media_opt_(5 * 30),
86 framedrop_percent_all_(5 * 30),
87 experiment_enabled_(QualityScalingExperiment::Enabled()),
88 observed_enough_frames_(false),
89 min_frames_needed_(
90 QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
91 kMinFramesNeededToScale)),
92 initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
93 .InitialScaleFactor()
94 .value_or(kSamplePeriodScaleFactor)),
95 scale_factor_(
96 QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()),
97 adapt_called_(false),
98 adapt_failed_(false) {
99 RTC_DCHECK_RUN_ON(&task_checker_);
100 if (experiment_enabled_) {
101 config_ = QualityScalingExperiment::GetConfig();
102 qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
103 qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
104 }
105 RTC_DCHECK(observer_ != nullptr);
106 check_qp_task_ = RepeatingTaskHandle::DelayedStart(
107 TaskQueueBase::Current(), TimeDelta::Millis(GetSamplingPeriodMs()),
108 [this]() {
109 CheckQp();
110 return TimeDelta::Millis(GetSamplingPeriodMs());
111 });
112 RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
113 << ", high: " << thresholds_.high;
114 }
115
~QualityScaler()116 QualityScaler::~QualityScaler() {
117 RTC_DCHECK_RUN_ON(&task_checker_);
118 check_qp_task_.Stop();
119 }
120
GetSamplingPeriodMs() const121 int64_t QualityScaler::GetSamplingPeriodMs() const {
122 RTC_DCHECK_RUN_ON(&task_checker_);
123 if (fast_rampup_) {
124 return sampling_period_ms_;
125 }
126 if (experiment_enabled_ && !observed_enough_frames_) {
127 // Use half the interval while waiting for enough frames.
128 return sampling_period_ms_ / 2;
129 }
130 if (adapt_failed_) {
131 // Check shortly again.
132 return sampling_period_ms_ / 8;
133 }
134 if (scale_factor_ && !adapt_called_) {
135 // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
136 return sampling_period_ms_ * scale_factor_.value();
137 }
138 return sampling_period_ms_ * initial_scale_factor_;
139 }
140
SetQpThresholds(VideoEncoder::QpThresholds thresholds)141 void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
142 RTC_DCHECK_RUN_ON(&task_checker_);
143 thresholds_ = thresholds;
144 }
145
ReportDroppedFrameByMediaOpt()146 void QualityScaler::ReportDroppedFrameByMediaOpt() {
147 RTC_DCHECK_RUN_ON(&task_checker_);
148 framedrop_percent_media_opt_.AddSample(100);
149 framedrop_percent_all_.AddSample(100);
150 }
151
ReportDroppedFrameByEncoder()152 void QualityScaler::ReportDroppedFrameByEncoder() {
153 RTC_DCHECK_RUN_ON(&task_checker_);
154 framedrop_percent_all_.AddSample(100);
155 }
156
ReportQp(int qp,int64_t time_sent_us)157 void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
158 RTC_DCHECK_RUN_ON(&task_checker_);
159 framedrop_percent_media_opt_.AddSample(0);
160 framedrop_percent_all_.AddSample(0);
161 average_qp_.AddSample(qp);
162 if (qp_smoother_high_)
163 qp_smoother_high_->Add(qp, time_sent_us);
164 if (qp_smoother_low_)
165 qp_smoother_low_->Add(qp, time_sent_us);
166 }
167
QpFastFilterLow() const168 bool QualityScaler::QpFastFilterLow() const {
169 RTC_DCHECK_RUN_ON(&task_checker_);
170 size_t num_frames = config_.use_all_drop_reasons
171 ? framedrop_percent_all_.Size()
172 : framedrop_percent_media_opt_.Size();
173 const size_t kMinNumFrames = 10;
174 if (num_frames < kMinNumFrames) {
175 return false; // Wait for more frames before making a decision.
176 }
177 absl::optional<int> avg_qp_high = qp_smoother_high_
178 ? qp_smoother_high_->GetAvg()
179 : average_qp_.GetAverageRoundedDown();
180 return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
181 }
182
CheckQp()183 void QualityScaler::CheckQp() {
184 RTC_DCHECK_RUN_ON(&task_checker_);
185 // Should be set through InitEncode -> Should be set by now.
186 RTC_DCHECK_GE(thresholds_.low, 0);
187 adapt_failed_ = false;
188 adapt_called_ = false;
189
190 // If we have not observed at least this many frames we can't make a good
191 // scaling decision.
192 const size_t frames = config_.use_all_drop_reasons
193 ? framedrop_percent_all_.Size()
194 : framedrop_percent_media_opt_.Size();
195 if (frames < min_frames_needed_) {
196 observed_enough_frames_ = false;
197 return;
198 }
199 observed_enough_frames_ = true;
200
201 // Check if we should scale down due to high frame drop.
202 const absl::optional<int> drop_rate =
203 config_.use_all_drop_reasons
204 ? framedrop_percent_all_.GetAverageRoundedDown()
205 : framedrop_percent_media_opt_.GetAverageRoundedDown();
206 if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
207 RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
208 ReportQpHigh();
209 return;
210 }
211
212 // Check if we should scale up or down based on QP.
213 const absl::optional<int> avg_qp_high =
214 qp_smoother_high_ ? qp_smoother_high_->GetAvg()
215 : average_qp_.GetAverageRoundedDown();
216 const absl::optional<int> avg_qp_low =
217 qp_smoother_low_ ? qp_smoother_low_->GetAvg()
218 : average_qp_.GetAverageRoundedDown();
219 if (avg_qp_high && avg_qp_low) {
220 RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
221 << *avg_qp_low << ").";
222 if (*avg_qp_high > thresholds_.high) {
223 ReportQpHigh();
224 return;
225 }
226 if (*avg_qp_low <= thresholds_.low) {
227 // QP has been low. We want to try a higher resolution.
228 ReportQpLow();
229 return;
230 }
231 }
232 }
233
ReportQpLow()234 void QualityScaler::ReportQpLow() {
235 RTC_DCHECK_RUN_ON(&task_checker_);
236 ClearSamples();
237 observer_->AdaptUp(AdaptationObserverInterface::AdaptReason::kQuality);
238 adapt_called_ = true;
239 }
240
ReportQpHigh()241 void QualityScaler::ReportQpHigh() {
242 RTC_DCHECK_RUN_ON(&task_checker_);
243
244 if (observer_->AdaptDown(
245 AdaptationObserverInterface::AdaptReason::kQuality)) {
246 ClearSamples();
247 } else {
248 adapt_failed_ = true;
249 }
250
251 // If we've scaled down, wait longer before scaling up again.
252 if (fast_rampup_) {
253 fast_rampup_ = false;
254 }
255 adapt_called_ = true;
256 }
257
ClearSamples()258 void QualityScaler::ClearSamples() {
259 RTC_DCHECK_RUN_ON(&task_checker_);
260 framedrop_percent_media_opt_.Reset();
261 framedrop_percent_all_.Reset();
262 average_qp_.Reset();
263 if (qp_smoother_high_)
264 qp_smoother_high_->Reset();
265 if (qp_smoother_low_)
266 qp_smoother_low_->Reset();
267 }
268 } // namespace webrtc
269