1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/utility/quality_scaler.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "api/video/video_adaptation_reason.h"
17 #include "rtc_base/checks.h"
18 #include "rtc_base/experiments/quality_scaler_settings.h"
19 #include "rtc_base/logging.h"
20 #include "rtc_base/numerics/exp_filter.h"
21 #include "rtc_base/task_queue.h"
22 #include "rtc_base/task_utils/to_queued_task.h"
23 #include "rtc_base/weak_ptr.h"
24
25 // TODO(kthelgason): Some versions of Android have issues with log2.
26 // See https://code.google.com/p/android/issues/detail?id=212634 for details
27 #if defined(WEBRTC_ANDROID)
28 #define log2(x) (log(x) / log(2))
29 #endif
30
31 namespace webrtc {
32
33 namespace {
34 // TODO(nisse): Delete, delegate to encoders.
35 // Threshold constant used until first downscale (to permit fast rampup).
36 static const int kMeasureMs = 2000;
37 static const float kSamplePeriodScaleFactor = 2.5;
38 static const int kFramedropPercentThreshold = 60;
39 static const size_t kMinFramesNeededToScale = 2 * 30;
40
41 } // namespace
42
43 class QualityScaler::QpSmoother {
44 public:
QpSmoother(float alpha)45 explicit QpSmoother(float alpha)
46 : alpha_(alpha),
47 // The initial value of last_sample_ms doesn't matter since the smoother
48 // will ignore the time delta for the first update.
49 last_sample_ms_(0),
50 smoother_(alpha) {}
51
GetAvg() const52 absl::optional<int> GetAvg() const {
53 float value = smoother_.filtered();
54 if (value == rtc::ExpFilter::kValueUndefined) {
55 return absl::nullopt;
56 }
57 return static_cast<int>(value);
58 }
59
Add(float sample,int64_t time_sent_us)60 void Add(float sample, int64_t time_sent_us) {
61 int64_t now_ms = time_sent_us / 1000;
62 smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
63 last_sample_ms_ = now_ms;
64 }
65
Reset()66 void Reset() { smoother_.Reset(alpha_); }
67
68 private:
69 const float alpha_;
70 int64_t last_sample_ms_;
71 rtc::ExpFilter smoother_;
72 };
73
74 // The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
75 // task will either run to completion and trigger a new task being queued, or it
76 // will be destroyed because the QualityScaler is destroyed.
77 //
78 // When high or low QP is reported, the task will be pending until a callback is
79 // invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
80 // asynchronously and prevents checking for QP until the stream has potentially
81 // been reconfigured.
82 class QualityScaler::CheckQpTask {
83 public:
84 // The result of one CheckQpTask may influence the delay of the next
85 // CheckQpTask.
86 struct Result {
87 bool observed_enough_frames = false;
88 bool qp_usage_reported = false;
89 };
90
CheckQpTask(QualityScaler * quality_scaler,Result previous_task_result)91 CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
92 : quality_scaler_(quality_scaler),
93 state_(State::kNotStarted),
94 previous_task_result_(previous_task_result),
95 weak_ptr_factory_(this) {}
96
StartDelayedTask()97 void StartDelayedTask() {
98 RTC_DCHECK_EQ(state_, State::kNotStarted);
99 state_ = State::kCheckingQp;
100 TaskQueueBase::Current()->PostDelayedTask(
101 ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
102 if (!this_weak_ptr) {
103 // The task has been cancelled through destruction.
104 return;
105 }
106 RTC_DCHECK_EQ(state_, State::kCheckingQp);
107 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
108 switch (quality_scaler_->CheckQp()) {
109 case QualityScaler::CheckQpResult::kInsufficientSamples: {
110 result_.observed_enough_frames = false;
111 // After this line, |this| may be deleted.
112 break;
113 }
114 case QualityScaler::CheckQpResult::kNormalQp: {
115 result_.observed_enough_frames = true;
116 break;
117 }
118 case QualityScaler::CheckQpResult::kHighQp: {
119 result_.observed_enough_frames = true;
120 result_.qp_usage_reported = true;
121 quality_scaler_->fast_rampup_ = false;
122 quality_scaler_->handler_->OnReportQpUsageHigh();
123 quality_scaler_->ClearSamples();
124 break;
125 }
126 case QualityScaler::CheckQpResult::kLowQp: {
127 result_.observed_enough_frames = true;
128 result_.qp_usage_reported = true;
129 quality_scaler_->handler_->OnReportQpUsageLow();
130 quality_scaler_->ClearSamples();
131 break;
132 }
133 }
134 state_ = State::kCompleted;
135 // Starting the next task deletes the pending task. After this line,
136 // |this| has been deleted.
137 quality_scaler_->StartNextCheckQpTask();
138 }),
139 GetCheckingQpDelayMs());
140 }
141
HasCompletedTask() const142 bool HasCompletedTask() const { return state_ == State::kCompleted; }
143
result() const144 Result result() const {
145 RTC_DCHECK(HasCompletedTask());
146 return result_;
147 }
148
149 private:
150 enum class State {
151 kNotStarted,
152 kCheckingQp,
153 kCompleted,
154 };
155
156 // Determines the sampling period of CheckQpTasks.
GetCheckingQpDelayMs() const157 int64_t GetCheckingQpDelayMs() const {
158 RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
159 if (quality_scaler_->fast_rampup_) {
160 return quality_scaler_->sampling_period_ms_;
161 }
162 if (quality_scaler_->experiment_enabled_ &&
163 !previous_task_result_.observed_enough_frames) {
164 // Use half the interval while waiting for enough frames.
165 return quality_scaler_->sampling_period_ms_ / 2;
166 }
167 if (quality_scaler_->scale_factor_ &&
168 !previous_task_result_.qp_usage_reported) {
169 // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
170 return quality_scaler_->sampling_period_ms_ *
171 quality_scaler_->scale_factor_.value();
172 }
173 return quality_scaler_->sampling_period_ms_ *
174 quality_scaler_->initial_scale_factor_;
175 }
176
177 QualityScaler* const quality_scaler_;
178 State state_;
179 const Result previous_task_result_;
180 Result result_;
181
182 rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
183 };
184
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds)185 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
186 VideoEncoder::QpThresholds thresholds)
187 : QualityScaler(handler, thresholds, kMeasureMs) {}
188
189 // Protected ctor, should not be called directly.
QualityScaler(QualityScalerQpUsageHandlerInterface * handler,VideoEncoder::QpThresholds thresholds,int64_t default_sampling_period_ms)190 QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
191 VideoEncoder::QpThresholds thresholds,
192 int64_t default_sampling_period_ms)
193 : handler_(handler),
194 thresholds_(thresholds),
195 sampling_period_ms_(QualityScalerSettings::ParseFromFieldTrials()
196 .SamplingPeriodMs()
197 .value_or(default_sampling_period_ms)),
198 fast_rampup_(true),
199 // Arbitrarily choose size based on 30 fps for 5 seconds.
200 average_qp_(QualityScalerSettings::ParseFromFieldTrials()
201 .AverageQpWindow()
202 .value_or(5 * 30)),
203 framedrop_percent_media_opt_(5 * 30),
204 framedrop_percent_all_(5 * 30),
205 experiment_enabled_(QualityScalingExperiment::Enabled()),
206 min_frames_needed_(
207 QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
208 kMinFramesNeededToScale)),
209 initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
210 .InitialScaleFactor()
211 .value_or(kSamplePeriodScaleFactor)),
212 scale_factor_(
213 QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
214 RTC_DCHECK_RUN_ON(&task_checker_);
215 if (experiment_enabled_) {
216 config_ = QualityScalingExperiment::GetConfig();
217 qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
218 qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
219 }
220 RTC_DCHECK(handler_ != nullptr);
221 StartNextCheckQpTask();
222 RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
223 << ", high: " << thresholds_.high;
224 }
225
~QualityScaler()226 QualityScaler::~QualityScaler() {
227 RTC_DCHECK_RUN_ON(&task_checker_);
228 }
229
StartNextCheckQpTask()230 void QualityScaler::StartNextCheckQpTask() {
231 RTC_DCHECK_RUN_ON(&task_checker_);
232 RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
233 << "A previous CheckQpTask has not completed yet!";
234 CheckQpTask::Result previous_task_result;
235 if (pending_qp_task_) {
236 previous_task_result = pending_qp_task_->result();
237 }
238 pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
239 pending_qp_task_->StartDelayedTask();
240 }
241
SetQpThresholds(VideoEncoder::QpThresholds thresholds)242 void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
243 RTC_DCHECK_RUN_ON(&task_checker_);
244 thresholds_ = thresholds;
245 }
246
ReportDroppedFrameByMediaOpt()247 void QualityScaler::ReportDroppedFrameByMediaOpt() {
248 RTC_DCHECK_RUN_ON(&task_checker_);
249 framedrop_percent_media_opt_.AddSample(100);
250 framedrop_percent_all_.AddSample(100);
251 }
252
ReportDroppedFrameByEncoder()253 void QualityScaler::ReportDroppedFrameByEncoder() {
254 RTC_DCHECK_RUN_ON(&task_checker_);
255 framedrop_percent_all_.AddSample(100);
256 }
257
ReportQp(int qp,int64_t time_sent_us)258 void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
259 RTC_DCHECK_RUN_ON(&task_checker_);
260 framedrop_percent_media_opt_.AddSample(0);
261 framedrop_percent_all_.AddSample(0);
262 average_qp_.AddSample(qp);
263 if (qp_smoother_high_)
264 qp_smoother_high_->Add(qp, time_sent_us);
265 if (qp_smoother_low_)
266 qp_smoother_low_->Add(qp, time_sent_us);
267 }
268
QpFastFilterLow() const269 bool QualityScaler::QpFastFilterLow() const {
270 RTC_DCHECK_RUN_ON(&task_checker_);
271 size_t num_frames = config_.use_all_drop_reasons
272 ? framedrop_percent_all_.Size()
273 : framedrop_percent_media_opt_.Size();
274 const size_t kMinNumFrames = 10;
275 if (num_frames < kMinNumFrames) {
276 return false; // Wait for more frames before making a decision.
277 }
278 absl::optional<int> avg_qp_high = qp_smoother_high_
279 ? qp_smoother_high_->GetAvg()
280 : average_qp_.GetAverageRoundedDown();
281 return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
282 }
283
CheckQp() const284 QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
285 RTC_DCHECK_RUN_ON(&task_checker_);
286 // Should be set through InitEncode -> Should be set by now.
287 RTC_DCHECK_GE(thresholds_.low, 0);
288
289 // If we have not observed at least this many frames we can't make a good
290 // scaling decision.
291 const size_t frames = config_.use_all_drop_reasons
292 ? framedrop_percent_all_.Size()
293 : framedrop_percent_media_opt_.Size();
294 if (frames < min_frames_needed_) {
295 return CheckQpResult::kInsufficientSamples;
296 }
297
298 // Check if we should scale down due to high frame drop.
299 const absl::optional<int> drop_rate =
300 config_.use_all_drop_reasons
301 ? framedrop_percent_all_.GetAverageRoundedDown()
302 : framedrop_percent_media_opt_.GetAverageRoundedDown();
303 if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
304 RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
305 return CheckQpResult::kHighQp;
306 }
307
308 // Check if we should scale up or down based on QP.
309 const absl::optional<int> avg_qp_high =
310 qp_smoother_high_ ? qp_smoother_high_->GetAvg()
311 : average_qp_.GetAverageRoundedDown();
312 const absl::optional<int> avg_qp_low =
313 qp_smoother_low_ ? qp_smoother_low_->GetAvg()
314 : average_qp_.GetAverageRoundedDown();
315 if (avg_qp_high && avg_qp_low) {
316 RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
317 << *avg_qp_low << ").";
318 if (*avg_qp_high > thresholds_.high) {
319 return CheckQpResult::kHighQp;
320 }
321 if (*avg_qp_low <= thresholds_.low) {
322 // QP has been low. We want to try a higher resolution.
323 return CheckQpResult::kLowQp;
324 }
325 }
326 return CheckQpResult::kNormalQp;
327 }
328
ClearSamples()329 void QualityScaler::ClearSamples() {
330 RTC_DCHECK_RUN_ON(&task_checker_);
331 framedrop_percent_media_opt_.Reset();
332 framedrop_percent_all_.Reset();
333 average_qp_.Reset();
334 if (qp_smoother_high_)
335 qp_smoother_high_->Reset();
336 if (qp_smoother_low_)
337 qp_smoother_low_->Reset();
338 }
339
~QualityScalerQpUsageHandlerInterface()340 QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
341
342 } // namespace webrtc
343