1 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
2 *
3 * Use of this source code is governed by a BSD-style license
4 * that can be found in the LICENSE file in the root of the source
5 * tree. An additional intellectual property rights grant can be found
6 * in the file PATENTS. All contributing project authors may
7 * be found in the AUTHORS file in the root of the source tree.
8 */
9
10 #include "modules/video_coding/codecs/vp8/screenshare_layers.h"
11
12 #include <stdlib.h>
13
14 #include <algorithm>
15 #include <memory>
16
17 #include "modules/video_coding/include/video_codec_interface.h"
18 #include "rtc_base/checks.h"
19 #include "rtc_base/logging.h"
20 #include "system_wrappers/include/clock.h"
21 #include "system_wrappers/include/metrics.h"
22 #include "vpx/vp8cx.h"
23 #include "vpx/vpx_encoder.h"
24
25 namespace webrtc {
26
27 static const int kOneSecond90Khz = 90000;
28 static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
29 static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
30 static const int kQpDeltaThresholdForSync = 8;
31 static const int kMinBitrateKbpsForQpBoost = 500;
32
33 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
34 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
35
36 constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
37
38 // Always emit a frame with certain interval, even if bitrate targets have
39 // been exceeded. This prevents needless keyframe requests.
40 const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
41
Create(int simulcast_id,int num_temporal_layers,uint8_t initial_tl0_pic_idx) const42 webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create(
43 int simulcast_id,
44 int num_temporal_layers,
45 uint8_t initial_tl0_pic_idx) const {
46 webrtc::TemporalLayers* tl;
47 if (simulcast_id == 0) {
48 tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
49 webrtc::Clock::GetRealTimeClock());
50 } else {
51 TemporalLayersFactory rt_tl_factory;
52 tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand());
53 }
54 if (listener_)
55 listener_->OnTemporalLayersCreated(simulcast_id, tl);
56 return tl;
57 }
58
59 std::unique_ptr<webrtc::TemporalLayersChecker>
CreateChecker(int simulcast_id,int temporal_layers,uint8_t initial_tl0_pic_idx) const60 ScreenshareTemporalLayersFactory::CreateChecker(
61 int simulcast_id,
62 int temporal_layers,
63 uint8_t initial_tl0_pic_idx) const {
64 webrtc::TemporalLayersChecker* tlc;
65 if (simulcast_id == 0) {
66 tlc =
67 new webrtc::TemporalLayersChecker(temporal_layers, initial_tl0_pic_idx);
68 } else {
69 TemporalLayersFactory rt_tl_factory;
70 return rt_tl_factory.CreateChecker(simulcast_id, temporal_layers,
71 initial_tl0_pic_idx);
72 }
73 return std::unique_ptr<webrtc::TemporalLayersChecker>(tlc);
74 }
75
ScreenshareLayers(int num_temporal_layers,uint8_t initial_tl0_pic_idx,Clock * clock)76 ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
77 uint8_t initial_tl0_pic_idx,
78 Clock* clock)
79 : clock_(clock),
80 number_of_temporal_layers_(
81 std::min(kMaxNumTemporalLayers, num_temporal_layers)),
82 last_base_layer_sync_(false),
83 tl0_pic_idx_(initial_tl0_pic_idx),
84 active_layer_(-1),
85 last_timestamp_(-1),
86 last_sync_timestamp_(-1),
87 last_emitted_tl0_timestamp_(-1),
88 min_qp_(-1),
89 max_qp_(-1),
90 max_debt_bytes_(0),
91 encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
92 bitrate_updated_(false) {
93 RTC_CHECK_GT(number_of_temporal_layers_, 0);
94 RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
95 }
96
~ScreenshareLayers()97 ScreenshareLayers::~ScreenshareLayers() {
98 UpdateHistograms();
99 }
100
Tl0PicIdx() const101 uint8_t ScreenshareLayers::Tl0PicIdx() const {
102 return tl0_pic_idx_;
103 }
104
UpdateLayerConfig(uint32_t timestamp)105 TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig(
106 uint32_t timestamp) {
107 if (number_of_temporal_layers_ <= 1) {
108 // No flags needed for 1 layer screenshare.
109 // TODO(pbos): Consider updating only last, and not all buffers.
110 TemporalLayers::FrameConfig tl_config(
111 kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate);
112 return tl_config;
113 }
114
115 const int64_t now_ms = clock_->TimeInMilliseconds();
116 if (target_framerate_.value_or(0) > 0 &&
117 encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
118 // Max framerate exceeded, drop frame.
119 return TemporalLayers::FrameConfig(kNone, kNone, kNone);
120 }
121
122 if (stats_.first_frame_time_ms_ == -1)
123 stats_.first_frame_time_ms_ = now_ms;
124
125 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
126 int64_t ts_diff;
127 if (last_timestamp_ == -1) {
128 ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
129 } else {
130 ts_diff = unwrapped_timestamp - last_timestamp_;
131 }
132 // Make sure both frame droppers leak out bits.
133 layers_[0].UpdateDebt(ts_diff / 90);
134 layers_[1].UpdateDebt(ts_diff / 90);
135 last_timestamp_ = timestamp;
136
137 TemporalLayerState layer_state = TemporalLayerState::kDrop;
138
139 if (active_layer_ == -1 ||
140 layers_[active_layer_].state != TemporalLayer::State::kDropped) {
141 if (last_emitted_tl0_timestamp_ != -1 &&
142 (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
143 kMaxFrameIntervalMs) {
144 // Too long time has passed since the last frame was emitted, cancel
145 // enough debt to allow a single frame.
146 layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
147 }
148 if (layers_[0].debt_bytes_ > max_debt_bytes_) {
149 // Must drop TL0, encode TL1 instead.
150 if (layers_[1].debt_bytes_ > max_debt_bytes_) {
151 // Must drop both TL0 and TL1.
152 active_layer_ = -1;
153 } else {
154 active_layer_ = 1;
155 }
156 } else {
157 active_layer_ = 0;
158 }
159 }
160
161 switch (active_layer_) {
162 case 0:
163 layer_state = TemporalLayerState::kTl0;
164 last_emitted_tl0_timestamp_ = unwrapped_timestamp;
165 break;
166 case 1:
167 if (layers_[1].state != TemporalLayer::State::kDropped) {
168 if (TimeToSync(unwrapped_timestamp)) {
169 last_sync_timestamp_ = unwrapped_timestamp;
170 layer_state = TemporalLayerState::kTl1Sync;
171 } else {
172 layer_state = TemporalLayerState::kTl1;
173 }
174 } else {
175 layer_state = last_sync_timestamp_ == unwrapped_timestamp
176 ? TemporalLayerState::kTl1Sync
177 : TemporalLayerState::kTl1;
178 }
179 break;
180 case -1:
181 layer_state = TemporalLayerState::kDrop;
182 ++stats_.num_dropped_frames_;
183 break;
184 default:
185 RTC_NOTREACHED();
186 }
187
188 TemporalLayers::FrameConfig tl_config;
189 // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
190 // layers.
191 switch (layer_state) {
192 case TemporalLayerState::kDrop:
193 tl_config = TemporalLayers::FrameConfig(kNone, kNone, kNone);
194 break;
195 case TemporalLayerState::kTl0:
196 // TL0 only references and updates 'last'.
197 tl_config =
198 TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone);
199 tl_config.packetizer_temporal_idx = 0;
200 break;
201 case TemporalLayerState::kTl1:
202 // TL1 references both 'last' and 'golden' but only updates 'golden'.
203 tl_config =
204 TemporalLayers::FrameConfig(kReference, kReferenceAndUpdate, kNone);
205 tl_config.packetizer_temporal_idx = 1;
206 break;
207 case TemporalLayerState::kTl1Sync:
208 // Predict from only TL0 to allow participants to switch to the high
209 // bitrate stream. Updates 'golden' so that TL1 can continue to refer to
210 // and update 'golden' from this point on.
211 tl_config = TemporalLayers::FrameConfig(kReference, kUpdate, kNone);
212 tl_config.packetizer_temporal_idx = 1;
213 break;
214 }
215
216 tl_config.layer_sync = layer_state == TemporalLayerState::kTl1Sync;
217 return tl_config;
218 }
219
OnRatesUpdated(int bitrate_kbps,int max_bitrate_kbps,int framerate)220 std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
221 int max_bitrate_kbps,
222 int framerate) {
223 RTC_DCHECK_GT(framerate, 0);
224 if (!target_framerate_) {
225 // First OnRatesUpdated() is called during construction, with the configured
226 // targets as parameters.
227 target_framerate_.emplace(framerate);
228 capture_framerate_ = target_framerate_;
229 bitrate_updated_ = true;
230 } else {
231 bitrate_updated_ =
232 bitrate_kbps != static_cast<int>(layers_[0].target_rate_kbps_) ||
233 max_bitrate_kbps != static_cast<int>(layers_[1].target_rate_kbps_) ||
234 (capture_framerate_ &&
235 framerate != static_cast<int>(*capture_framerate_));
236 if (framerate < 0) {
237 capture_framerate_.reset();
238 } else {
239 capture_framerate_.emplace(framerate);
240 }
241 }
242
243 layers_[0].target_rate_kbps_ = bitrate_kbps;
244 layers_[1].target_rate_kbps_ = max_bitrate_kbps;
245
246 std::vector<uint32_t> allocation;
247 allocation.push_back(bitrate_kbps);
248 if (max_bitrate_kbps > bitrate_kbps)
249 allocation.push_back(max_bitrate_kbps - bitrate_kbps);
250 return allocation;
251 }
252
FrameEncoded(unsigned int size,int qp)253 void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) {
254 if (size > 0)
255 encode_framerate_.Update(1, clock_->TimeInMilliseconds());
256
257 if (number_of_temporal_layers_ == 1)
258 return;
259
260 RTC_DCHECK_NE(-1, active_layer_);
261 if (size == 0) {
262 layers_[active_layer_].state = TemporalLayer::State::kDropped;
263 ++stats_.num_overshoots_;
264 return;
265 }
266
267 if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
268 layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
269 }
270
271 if (qp != -1)
272 layers_[active_layer_].last_qp = qp;
273
274 if (active_layer_ == 0) {
275 layers_[0].debt_bytes_ += size;
276 layers_[1].debt_bytes_ += size;
277 ++stats_.num_tl0_frames_;
278 stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
279 stats_.tl0_qp_sum_ += qp;
280 } else if (active_layer_ == 1) {
281 layers_[1].debt_bytes_ += size;
282 ++stats_.num_tl1_frames_;
283 stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
284 stats_.tl1_qp_sum_ += qp;
285 }
286 }
287
PopulateCodecSpecific(bool frame_is_keyframe,const TemporalLayers::FrameConfig & tl_config,CodecSpecificInfoVP8 * vp8_info,uint32_t timestamp)288 void ScreenshareLayers::PopulateCodecSpecific(
289 bool frame_is_keyframe,
290 const TemporalLayers::FrameConfig& tl_config,
291 CodecSpecificInfoVP8* vp8_info,
292 uint32_t timestamp) {
293 if (number_of_temporal_layers_ == 1) {
294 vp8_info->temporalIdx = kNoTemporalIdx;
295 vp8_info->layerSync = false;
296 vp8_info->tl0PicIdx = kNoTl0PicIdx;
297 } else {
298 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
299 vp8_info->temporalIdx = tl_config.packetizer_temporal_idx;
300 vp8_info->layerSync = tl_config.layer_sync;
301 if (frame_is_keyframe) {
302 vp8_info->temporalIdx = 0;
303 last_sync_timestamp_ = unwrapped_timestamp;
304 vp8_info->layerSync = true;
305 } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
306 // Regardless of pattern the frame after a base layer sync will always
307 // be a layer sync.
308 last_sync_timestamp_ = unwrapped_timestamp;
309 vp8_info->layerSync = true;
310 }
311 if (vp8_info->temporalIdx == 0) {
312 tl0_pic_idx_++;
313 }
314 last_base_layer_sync_ = frame_is_keyframe;
315 vp8_info->tl0PicIdx = tl0_pic_idx_;
316 }
317 }
318
TimeToSync(int64_t timestamp) const319 bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
320 RTC_DCHECK_EQ(1, active_layer_);
321 RTC_DCHECK_NE(-1, layers_[0].last_qp);
322 if (layers_[1].last_qp == -1) {
323 // First frame in TL1 should only depend on TL0 since there are no
324 // previous frames in TL1.
325 return true;
326 }
327
328 RTC_DCHECK_NE(-1, last_sync_timestamp_);
329 int64_t timestamp_diff = timestamp - last_sync_timestamp_;
330 if (timestamp_diff > kMaxTimeBetweenSyncs) {
331 // After a certain time, force a sync frame.
332 return true;
333 } else if (timestamp_diff < kMinTimeBetweenSyncs) {
334 // If too soon from previous sync frame, don't issue a new one.
335 return false;
336 }
337 // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
338 // large.
339 if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
340 return true;
341 return false;
342 }
343
GetCodecTargetBitrateKbps() const344 uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
345 uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
346
347 if (number_of_temporal_layers_ > 1) {
348 // Calculate a codec target bitrate. This may be higher than TL0, gaining
349 // quality at the expense of frame rate at TL0. Constraints:
350 // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
351 // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
352 target_bitrate_kbps =
353 std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
354 layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
355 }
356
357 return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
358 }
359
UpdateConfiguration(vpx_codec_enc_cfg_t * cfg)360 bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
361 bool cfg_updated = false;
362 uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
363 if (bitrate_updated_ || cfg->rc_target_bitrate != target_bitrate_kbps) {
364 cfg->rc_target_bitrate = target_bitrate_kbps;
365
366 // Don't reconfigure qp limits during quality boost frames.
367 if (active_layer_ == -1 ||
368 layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
369 min_qp_ = cfg->rc_min_quantizer;
370 max_qp_ = cfg->rc_max_quantizer;
371 // After a dropped frame, a frame with max qp will be encoded and the
372 // quality will then ramp up from there. To boost the speed of recovery,
373 // encode the next frame with lower max qp, if there is sufficient
374 // bandwidth to do so without causing excessive delay.
375 // TL0 is the most important to improve since the errors in this layer
376 // will propagate to TL1.
377 // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
378 if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) {
379 layers_[0].enhanced_max_qp =
380 min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
381 layers_[1].enhanced_max_qp =
382 min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
383 } else {
384 layers_[0].enhanced_max_qp = -1;
385 layers_[1].enhanced_max_qp = -1;
386 }
387 }
388
389 if (capture_framerate_) {
390 int avg_frame_size =
391 (target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
392 // Allow max debt to be the size of a single optimal frame.
393 // TODO(sprang): Determine if this needs to be adjusted by some factor.
394 // (Lower values may cause more frame drops, higher may lead to queuing
395 // delays.)
396 max_debt_bytes_ = avg_frame_size;
397 }
398
399 bitrate_updated_ = false;
400 cfg_updated = true;
401 }
402
403 // Don't try to update boosts state if not active yet.
404 if (active_layer_ == -1)
405 return cfg_updated;
406
407 if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
408 return cfg_updated;
409
410 // If layer is in the quality boost state (following a dropped frame), update
411 // the configuration with the adjusted (lower) qp and set the state back to
412 // normal.
413 unsigned int adjusted_max_qp;
414 if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
415 layers_[active_layer_].enhanced_max_qp != -1) {
416 adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
417 layers_[active_layer_].state = TemporalLayer::State::kNormal;
418 } else {
419 adjusted_max_qp = max_qp_; // Set the normal max qp.
420 }
421
422 if (adjusted_max_qp == cfg->rc_max_quantizer)
423 return cfg_updated;
424
425 cfg->rc_max_quantizer = adjusted_max_qp;
426 cfg_updated = true;
427
428 return cfg_updated;
429 }
430
UpdateDebt(int64_t delta_ms)431 void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
432 uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
433 if (debt_reduction_bytes >= debt_bytes_) {
434 debt_bytes_ = 0;
435 } else {
436 debt_bytes_ -= debt_reduction_bytes;
437 }
438 }
439
UpdateHistograms()440 void ScreenshareLayers::UpdateHistograms() {
441 if (stats_.first_frame_time_ms_ == -1)
442 return;
443 int64_t duration_sec =
444 (clock_->TimeInMilliseconds() - stats_.first_frame_time_ms_ + 500) / 1000;
445 if (duration_sec >= metrics::kMinRunTimeInSeconds) {
446 RTC_HISTOGRAM_COUNTS_10000(
447 "WebRTC.Video.Screenshare.Layer0.FrameRate",
448 (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
449 RTC_HISTOGRAM_COUNTS_10000(
450 "WebRTC.Video.Screenshare.Layer1.FrameRate",
451 (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
452 int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
453 RTC_HISTOGRAM_COUNTS_10000(
454 "WebRTC.Video.Screenshare.FramesPerDrop",
455 (stats_.num_dropped_frames_ == 0
456 ? 0
457 : total_frames / stats_.num_dropped_frames_));
458 RTC_HISTOGRAM_COUNTS_10000(
459 "WebRTC.Video.Screenshare.FramesPerOvershoot",
460 (stats_.num_overshoots_ == 0 ? 0
461 : total_frames / stats_.num_overshoots_));
462 if (stats_.num_tl0_frames_ > 0) {
463 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
464 stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
465 RTC_HISTOGRAM_COUNTS_10000(
466 "WebRTC.Video.Screenshare.Layer0.TargetBitrate",
467 stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
468 }
469 if (stats_.num_tl1_frames_ > 0) {
470 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
471 stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
472 RTC_HISTOGRAM_COUNTS_10000(
473 "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
474 stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
475 }
476 }
477 }
478 } // namespace webrtc
479