1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
12 #ifdef RTC_ENABLE_VP9
13
14 #include "modules/video_coding/codecs/vp9/vp9_impl.h"
15
16 #include <algorithm>
17 #include <limits>
18 #include <utility>
19 #include <vector>
20
21 #include "absl/memory/memory.h"
22 #include "api/video/color_space.h"
23 #include "api/video/i010_buffer.h"
24 #include "common_video/include/video_frame_buffer.h"
25 #include "common_video/libyuv/include/webrtc_libyuv.h"
26 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
27 #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
28 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
29 #include "rtc_base/checks.h"
30 #include "rtc_base/experiments/rate_control_settings.h"
31 #include "rtc_base/keep_ref_until_done.h"
32 #include "rtc_base/logging.h"
33 #include "rtc_base/time_utils.h"
34 #include "rtc_base/trace_event.h"
35 #include "system_wrappers/include/field_trial.h"
36 #include "vpx/vp8cx.h"
37 #include "vpx/vp8dx.h"
38 #include "vpx/vpx_decoder.h"
39 #include "vpx/vpx_encoder.h"
40
41 namespace webrtc {
42
43 namespace {
44 // Maps from gof_idx to encoder internal reference frame buffer index. These
45 // maps work for 1,2 and 3 temporal layers with GOF length of 1,2 and 4 frames.
46 uint8_t kRefBufIdx[4] = {0, 0, 0, 1};
47 uint8_t kUpdBufIdx[4] = {0, 0, 1, 0};
48
49 // Maximum allowed PID difference for differnet per-layer frame-rate case.
50 const int kMaxAllowedPidDiff = 30;
51
52 constexpr double kLowRateFactor = 1.0;
53 constexpr double kHighRateFactor = 2.0;
54
55 // TODO(ilink): Tune these thresholds further.
56 // Selected using ConverenceMotion_1280_720_50.yuv clip.
57 // No toggling observed on any link capacity from 100-2000kbps.
58 // HD was reached consistently when link capacity was 1500kbps.
59 // Set resolutions are a bit more conservative than svc_config.cc sets, e.g.
60 // for 300kbps resolution converged to 270p instead of 360p.
61 constexpr int kLowVp9QpThreshold = 149;
62 constexpr int kHighVp9QpThreshold = 205;
63
64 // These settings correspond to the settings in vpx_codec_enc_cfg.
65 struct Vp9RateSettings {
66 uint32_t rc_undershoot_pct;
67 uint32_t rc_overshoot_pct;
68 uint32_t rc_buf_sz;
69 uint32_t rc_buf_optimal_sz;
70 uint32_t rc_dropframe_thresh;
71 };
72
73 // Only positive speeds, range for real-time coding currently is: 5 - 8.
74 // Lower means slower/better quality, higher means fastest/lower quality.
GetCpuSpeed(int width,int height)75 int GetCpuSpeed(int width, int height) {
76 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
77 return 8;
78 #else
79 // For smaller resolutions, use lower speed setting (get some coding gain at
80 // the cost of increased encoding complexity).
81 if (width * height <= 352 * 288)
82 return 5;
83 else
84 return 7;
85 #endif
86 }
87 // Helper class for extracting VP9 colorspace.
ExtractVP9ColorSpace(vpx_color_space_t space_t,vpx_color_range_t range_t,unsigned int bit_depth)88 ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t,
89 vpx_color_range_t range_t,
90 unsigned int bit_depth) {
91 ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kUnspecified;
92 ColorSpace::TransferID transfer = ColorSpace::TransferID::kUnspecified;
93 ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kUnspecified;
94 switch (space_t) {
95 case VPX_CS_BT_601:
96 case VPX_CS_SMPTE_170:
97 primaries = ColorSpace::PrimaryID::kSMPTE170M;
98 transfer = ColorSpace::TransferID::kSMPTE170M;
99 matrix = ColorSpace::MatrixID::kSMPTE170M;
100 break;
101 case VPX_CS_SMPTE_240:
102 primaries = ColorSpace::PrimaryID::kSMPTE240M;
103 transfer = ColorSpace::TransferID::kSMPTE240M;
104 matrix = ColorSpace::MatrixID::kSMPTE240M;
105 break;
106 case VPX_CS_BT_709:
107 primaries = ColorSpace::PrimaryID::kBT709;
108 transfer = ColorSpace::TransferID::kBT709;
109 matrix = ColorSpace::MatrixID::kBT709;
110 break;
111 case VPX_CS_BT_2020:
112 primaries = ColorSpace::PrimaryID::kBT2020;
113 switch (bit_depth) {
114 case 8:
115 transfer = ColorSpace::TransferID::kBT709;
116 break;
117 case 10:
118 transfer = ColorSpace::TransferID::kBT2020_10;
119 break;
120 default:
121 RTC_NOTREACHED();
122 break;
123 }
124 matrix = ColorSpace::MatrixID::kBT2020_NCL;
125 break;
126 case VPX_CS_SRGB:
127 primaries = ColorSpace::PrimaryID::kBT709;
128 transfer = ColorSpace::TransferID::kIEC61966_2_1;
129 matrix = ColorSpace::MatrixID::kBT709;
130 break;
131 default:
132 break;
133 }
134
135 ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
136 switch (range_t) {
137 case VPX_CR_STUDIO_RANGE:
138 range = ColorSpace::RangeID::kLimited;
139 break;
140 case VPX_CR_FULL_RANGE:
141 range = ColorSpace::RangeID::kFull;
142 break;
143 default:
144 break;
145 }
146 return ColorSpace(primaries, transfer, matrix, range);
147 }
148
GetActiveLayers(const VideoBitrateAllocation & allocation)149 std::pair<size_t, size_t> GetActiveLayers(
150 const VideoBitrateAllocation& allocation) {
151 for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) {
152 if (allocation.GetSpatialLayerSum(sl_idx) > 0) {
153 size_t last_layer = sl_idx + 1;
154 while (last_layer < kMaxSpatialLayers &&
155 allocation.GetSpatialLayerSum(last_layer) > 0) {
156 ++last_layer;
157 }
158 return std::make_pair(sl_idx, last_layer);
159 }
160 }
161 return {0, 0};
162 }
163
Interpolate(uint32_t low,uint32_t high,double bandwidth_headroom_factor)164 uint32_t Interpolate(uint32_t low,
165 uint32_t high,
166 double bandwidth_headroom_factor) {
167 RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor);
168 RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor);
169
170 // |factor| is between 0.0 and 1.0.
171 const double factor = bandwidth_headroom_factor - kLowRateFactor;
172
173 return static_cast<uint32_t>(((1.0 - factor) * low) + (factor * high) + 0.5);
174 }
175
GetRateSettings(double bandwidth_headroom_factor)176 Vp9RateSettings GetRateSettings(double bandwidth_headroom_factor) {
177 static const Vp9RateSettings low_settings{100u, 0u, 100u, 33u, 40u};
178 static const Vp9RateSettings high_settings{50u, 50u, 1000u, 700u, 5u};
179
180 if (bandwidth_headroom_factor <= kLowRateFactor) {
181 return low_settings;
182 } else if (bandwidth_headroom_factor >= kHighRateFactor) {
183 return high_settings;
184 }
185
186 Vp9RateSettings settings;
187 settings.rc_undershoot_pct =
188 Interpolate(low_settings.rc_undershoot_pct,
189 high_settings.rc_undershoot_pct, bandwidth_headroom_factor);
190 settings.rc_overshoot_pct =
191 Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct,
192 bandwidth_headroom_factor);
193 settings.rc_buf_sz =
194 Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz,
195 bandwidth_headroom_factor);
196 settings.rc_buf_optimal_sz =
197 Interpolate(low_settings.rc_buf_optimal_sz,
198 high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor);
199 settings.rc_dropframe_thresh =
200 Interpolate(low_settings.rc_dropframe_thresh,
201 high_settings.rc_dropframe_thresh, bandwidth_headroom_factor);
202 return settings;
203 }
204
UpdateRateSettings(vpx_codec_enc_cfg_t * config,const Vp9RateSettings & new_settings)205 void UpdateRateSettings(vpx_codec_enc_cfg_t* config,
206 const Vp9RateSettings& new_settings) {
207 config->rc_undershoot_pct = new_settings.rc_undershoot_pct;
208 config->rc_overshoot_pct = new_settings.rc_overshoot_pct;
209 config->rc_buf_sz = new_settings.rc_buf_sz;
210 config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz;
211 config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh;
212 }
213
214 } // namespace
215
EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt * pkt,void * user_data)216 void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
217 void* user_data) {
218 VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data);
219 enc->GetEncodedLayerFrame(pkt);
220 }
221
VP9EncoderImpl(const cricket::VideoCodec & codec)222 VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
223 : encoded_image_(),
224 encoded_complete_callback_(nullptr),
225 profile_(
226 ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)),
227 inited_(false),
228 timestamp_(0),
229 cpu_speed_(3),
230 rc_max_intra_target_(0),
231 encoder_(nullptr),
232 config_(nullptr),
233 raw_(nullptr),
234 input_image_(nullptr),
235 force_key_frame_(true),
236 pics_since_key_(0),
237 num_temporal_layers_(0),
238 num_spatial_layers_(0),
239 num_active_spatial_layers_(0),
240 first_active_layer_(0),
241 layer_deactivation_requires_key_frame_(
242 field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
243 is_svc_(false),
244 inter_layer_pred_(InterLayerPredMode::kOn),
245 external_ref_control_(false), // Set in InitEncode because of tests.
246 trusted_rate_controller_(RateControlSettings::ParseFromFieldTrials()
247 .LibvpxVp9TrustedRateController()),
248 dynamic_rate_settings_(
249 RateControlSettings::ParseFromFieldTrials().Vp9DynamicRateSettings()),
250 layer_buffering_(false),
251 full_superframe_drop_(true),
252 first_frame_in_picture_(true),
253 ss_info_needed_(false),
254 force_all_active_layers_(false),
255 num_cores_(0),
256 is_flexible_mode_(false),
257 variable_framerate_experiment_(ParseVariableFramerateConfig(
258 "WebRTC-VP9VariableFramerateScreenshare")),
259 variable_framerate_controller_(
260 variable_framerate_experiment_.framerate_limit),
261 quality_scaler_experiment_(
262 ParseQualityScalerConfig("WebRTC-VP9QualityScaler")),
263 num_steady_state_frames_(0),
264 config_changed_(true) {
265 codec_ = {};
266 memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
267 }
268
~VP9EncoderImpl()269 VP9EncoderImpl::~VP9EncoderImpl() {
270 Release();
271 }
272
SetFecControllerOverride(FecControllerOverride * fec_controller_override)273 void VP9EncoderImpl::SetFecControllerOverride(
274 FecControllerOverride* fec_controller_override) {
275 // Ignored.
276 }
277
Release()278 int VP9EncoderImpl::Release() {
279 int ret_val = WEBRTC_VIDEO_CODEC_OK;
280
281 if (encoder_ != nullptr) {
282 if (inited_) {
283 if (vpx_codec_destroy(encoder_)) {
284 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
285 }
286 }
287 delete encoder_;
288 encoder_ = nullptr;
289 }
290 if (config_ != nullptr) {
291 delete config_;
292 config_ = nullptr;
293 }
294 if (raw_ != nullptr) {
295 vpx_img_free(raw_);
296 raw_ = nullptr;
297 }
298 inited_ = false;
299 return ret_val;
300 }
301
ExplicitlyConfiguredSpatialLayers() const302 bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
303 // We check target_bitrate_bps of the 0th layer to see if the spatial layers
304 // (i.e. bitrates) were explicitly configured.
305 return codec_.spatialLayers[0].targetBitrate > 0;
306 }
307
SetSvcRates(const VideoBitrateAllocation & bitrate_allocation)308 bool VP9EncoderImpl::SetSvcRates(
309 const VideoBitrateAllocation& bitrate_allocation) {
310 std::pair<size_t, size_t> current_layers =
311 GetActiveLayers(current_bitrate_allocation_);
312 std::pair<size_t, size_t> new_layers = GetActiveLayers(bitrate_allocation);
313
314 const bool layer_activation_requires_key_frame =
315 inter_layer_pred_ == InterLayerPredMode::kOff ||
316 inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
317 const bool lower_layers_enabled = new_layers.first < current_layers.first;
318 const bool higher_layers_enabled = new_layers.second > current_layers.second;
319 const bool disabled_layers = new_layers.first > current_layers.first ||
320 new_layers.second < current_layers.second;
321
322 if (lower_layers_enabled ||
323 (higher_layers_enabled && layer_activation_requires_key_frame) ||
324 (disabled_layers && layer_deactivation_requires_key_frame_)) {
325 force_key_frame_ = true;
326 }
327
328 if (current_layers != new_layers) {
329 ss_info_needed_ = true;
330 }
331
332 config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
333
334 if (ExplicitlyConfiguredSpatialLayers()) {
335 for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
336 const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
337 config_->ss_target_bitrate[sl_idx] =
338 bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000;
339
340 for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) {
341 config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] =
342 bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
343 }
344
345 if (!was_layer_active) {
346 // Reset frame rate controller if layer is resumed after pause.
347 framerate_controller_[sl_idx].Reset();
348 }
349
350 framerate_controller_[sl_idx].SetTargetRate(
351 codec_.spatialLayers[sl_idx].maxFramerate);
352 }
353 } else {
354 float rate_ratio[VPX_MAX_LAYERS] = {0};
355 float total = 0;
356 for (int i = 0; i < num_spatial_layers_; ++i) {
357 if (svc_params_.scaling_factor_num[i] <= 0 ||
358 svc_params_.scaling_factor_den[i] <= 0) {
359 RTC_LOG(LS_ERROR) << "Scaling factors not specified!";
360 return false;
361 }
362 rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) /
363 svc_params_.scaling_factor_den[i];
364 total += rate_ratio[i];
365 }
366
367 for (int i = 0; i < num_spatial_layers_; ++i) {
368 RTC_CHECK_GT(total, 0);
369 config_->ss_target_bitrate[i] = static_cast<unsigned int>(
370 config_->rc_target_bitrate * rate_ratio[i] / total);
371 if (num_temporal_layers_ == 1) {
372 config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i];
373 } else if (num_temporal_layers_ == 2) {
374 config_->layer_target_bitrate[i * num_temporal_layers_] =
375 config_->ss_target_bitrate[i] * 2 / 3;
376 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
377 config_->ss_target_bitrate[i];
378 } else if (num_temporal_layers_ == 3) {
379 config_->layer_target_bitrate[i * num_temporal_layers_] =
380 config_->ss_target_bitrate[i] / 2;
381 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
382 config_->layer_target_bitrate[i * num_temporal_layers_] +
383 (config_->ss_target_bitrate[i] / 4);
384 config_->layer_target_bitrate[i * num_temporal_layers_ + 2] =
385 config_->ss_target_bitrate[i];
386 } else {
387 RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: "
388 << num_temporal_layers_;
389 return false;
390 }
391
392 framerate_controller_[i].SetTargetRate(codec_.maxFramerate);
393 }
394 }
395
396 num_active_spatial_layers_ = 0;
397 first_active_layer_ = 0;
398 bool seen_active_layer = false;
399 bool expect_no_more_active_layers = false;
400 for (int i = 0; i < num_spatial_layers_; ++i) {
401 if (config_->ss_target_bitrate[i] > 0) {
402 RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is "
403 "deactivated.";
404 if (!seen_active_layer) {
405 first_active_layer_ = i;
406 }
407 num_active_spatial_layers_ = i + 1;
408 seen_active_layer = true;
409 } else {
410 expect_no_more_active_layers = seen_active_layer;
411 }
412 }
413
414 if (higher_layers_enabled && !force_key_frame_) {
415 // Prohibit drop of all layers for the next frame, so newly enabled
416 // layer would have a valid spatial reference.
417 for (size_t i = 0; i < num_spatial_layers_; ++i) {
418 svc_drop_frame_.framedrop_thresh[i] = 0;
419 }
420 force_all_active_layers_ = true;
421 }
422
423 current_bitrate_allocation_ = bitrate_allocation;
424 config_changed_ = true;
425 return true;
426 }
427
SetRates(const RateControlParameters & parameters)428 void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) {
429 if (!inited_) {
430 RTC_LOG(LS_WARNING) << "SetRates() calll while uninitialzied.";
431 return;
432 }
433 if (encoder_->err) {
434 RTC_LOG(LS_WARNING) << "Encoder in error state: " << encoder_->err;
435 return;
436 }
437 if (parameters.framerate_fps < 1.0) {
438 RTC_LOG(LS_WARNING) << "Unsupported framerate: "
439 << parameters.framerate_fps;
440 return;
441 }
442
443 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
444
445 if (dynamic_rate_settings_) {
446 // Tweak rate control settings based on available network headroom.
447 UpdateRateSettings(
448 config_, GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
449 parameters.bitrate.get_sum_bps()));
450 }
451
452 bool res = SetSvcRates(parameters.bitrate);
453 RTC_DCHECK(res) << "Failed to set new bitrate allocation";
454 config_changed_ = true;
455 }
456
457 // TODO(eladalon): s/inst/codec_settings/g.
InitEncode(const VideoCodec * inst,const Settings & settings)458 int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
459 const Settings& settings) {
460 if (inst == nullptr) {
461 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
462 }
463 if (inst->maxFramerate < 1) {
464 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
465 }
466 // Allow zero to represent an unspecified maxBitRate
467 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
468 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
469 }
470 if (inst->width < 1 || inst->height < 1) {
471 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
472 }
473 if (settings.number_of_cores < 1) {
474 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
475 }
476 if (inst->VP9().numberOfTemporalLayers > 3) {
477 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
478 }
479 // libvpx probably does not support more than 3 spatial layers.
480 if (inst->VP9().numberOfSpatialLayers > 3) {
481 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
482 }
483
484 int ret_val = Release();
485 if (ret_val < 0) {
486 return ret_val;
487 }
488 if (encoder_ == nullptr) {
489 encoder_ = new vpx_codec_ctx_t;
490 }
491 if (config_ == nullptr) {
492 config_ = new vpx_codec_enc_cfg_t;
493 }
494 timestamp_ = 0;
495 if (&codec_ != inst) {
496 codec_ = *inst;
497 }
498
499 force_key_frame_ = true;
500 pics_since_key_ = 0;
501 num_cores_ = settings.number_of_cores;
502 num_spatial_layers_ = inst->VP9().numberOfSpatialLayers;
503 RTC_DCHECK_GT(num_spatial_layers_, 0);
504 num_temporal_layers_ = inst->VP9().numberOfTemporalLayers;
505 if (num_temporal_layers_ == 0) {
506 num_temporal_layers_ = 1;
507 }
508
509 framerate_controller_ = std::vector<FramerateController>(
510 num_spatial_layers_, FramerateController(codec_.maxFramerate));
511
512 is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
513
514 encoded_image_._completeFrame = true;
515 // Populate encoder configuration with default values.
516 if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) {
517 return WEBRTC_VIDEO_CODEC_ERROR;
518 }
519
520 vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE;
521 unsigned int bits_for_storage = 8;
522 switch (profile_) {
523 case VP9Profile::kProfile0:
524 img_fmt = VPX_IMG_FMT_I420;
525 bits_for_storage = 8;
526 config_->g_bit_depth = VPX_BITS_8;
527 config_->g_profile = 0;
528 config_->g_input_bit_depth = 8;
529 break;
530 case VP9Profile::kProfile1:
531 // Encoding of profile 1 is not implemented. It would require extended
532 // support for I444, I422, and I440 buffers.
533 RTC_NOTREACHED();
534 break;
535 case VP9Profile::kProfile2:
536 img_fmt = VPX_IMG_FMT_I42016;
537 bits_for_storage = 16;
538 config_->g_bit_depth = VPX_BITS_10;
539 config_->g_profile = 2;
540 config_->g_input_bit_depth = 10;
541 break;
542 }
543
544 // Creating a wrapper to the image - setting image data to nullptr. Actual
545 // pointer will be set in encode. Setting align to 1, as it is meaningless
546 // (actual memory is not allocated).
547 raw_ =
548 vpx_img_wrap(nullptr, img_fmt, codec_.width, codec_.height, 1, nullptr);
549 raw_->bit_depth = bits_for_storage;
550
551 config_->g_w = codec_.width;
552 config_->g_h = codec_.height;
553 config_->rc_target_bitrate = inst->startBitrate; // in kbit/s
554 config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0;
555 // Setting the time base of the codec.
556 config_->g_timebase.num = 1;
557 config_->g_timebase.den = 90000;
558 config_->g_lag_in_frames = 0; // 0- no frame lagging
559 config_->g_threads = 1;
560 // Rate control settings.
561 config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0;
562 config_->rc_end_usage = VPX_CBR;
563 config_->g_pass = VPX_RC_ONE_PASS;
564 config_->rc_min_quantizer =
565 codec_.mode == VideoCodecMode::kScreensharing ? 8 : 2;
566 config_->rc_max_quantizer = 52;
567 config_->rc_undershoot_pct = 50;
568 config_->rc_overshoot_pct = 50;
569 config_->rc_buf_initial_sz = 500;
570 config_->rc_buf_optimal_sz = 600;
571 config_->rc_buf_sz = 1000;
572 // Set the maximum target size of any key-frame.
573 rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz);
574 // Key-frame interval is enforced manually by this wrapper.
575 config_->kf_mode = VPX_KF_DISABLED;
576 // TODO(webm:1592): work-around for libvpx issue, as it can still
577 // put some key-frames at will even in VPX_KF_DISABLED kf_mode.
578 config_->kf_max_dist = inst->VP9().keyFrameInterval;
579 config_->kf_min_dist = config_->kf_max_dist;
580 if (quality_scaler_experiment_.enabled) {
581 // In that experiment webrtc wide quality scaler is used instead of libvpx
582 // internal scaler.
583 config_->rc_resize_allowed = 0;
584 } else {
585 config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
586 }
587 // Determine number of threads based on the image size and #cores.
588 config_->g_threads =
589 NumberOfThreads(config_->g_w, config_->g_h, settings.number_of_cores);
590
591 cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
592
593 is_flexible_mode_ = inst->VP9().flexibleMode;
594
595 inter_layer_pred_ = inst->VP9().interLayerPred;
596
597 if (num_spatial_layers_ > 1 &&
598 codec_.mode == VideoCodecMode::kScreensharing && !is_flexible_mode_) {
599 RTC_LOG(LS_ERROR) << "Flexible mode is required for screenshare with "
600 "several spatial layers";
601 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
602 }
603
604 // External reference control is required for different frame rate on spatial
605 // layers because libvpx generates rtp incompatible references in this case.
606 external_ref_control_ =
607 !field_trial::IsDisabled("WebRTC-Vp9ExternalRefCtrl") ||
608 (num_spatial_layers_ > 1 &&
609 codec_.mode == VideoCodecMode::kScreensharing) ||
610 inter_layer_pred_ == InterLayerPredMode::kOn;
611
612 if (num_temporal_layers_ == 1) {
613 gof_.SetGofInfoVP9(kTemporalStructureMode1);
614 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
615 config_->ts_number_layers = 1;
616 config_->ts_rate_decimator[0] = 1;
617 config_->ts_periodicity = 1;
618 config_->ts_layer_id[0] = 0;
619 } else if (num_temporal_layers_ == 2) {
620 gof_.SetGofInfoVP9(kTemporalStructureMode2);
621 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
622 config_->ts_number_layers = 2;
623 config_->ts_rate_decimator[0] = 2;
624 config_->ts_rate_decimator[1] = 1;
625 config_->ts_periodicity = 2;
626 config_->ts_layer_id[0] = 0;
627 config_->ts_layer_id[1] = 1;
628 } else if (num_temporal_layers_ == 3) {
629 gof_.SetGofInfoVP9(kTemporalStructureMode3);
630 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
631 config_->ts_number_layers = 3;
632 config_->ts_rate_decimator[0] = 4;
633 config_->ts_rate_decimator[1] = 2;
634 config_->ts_rate_decimator[2] = 1;
635 config_->ts_periodicity = 4;
636 config_->ts_layer_id[0] = 0;
637 config_->ts_layer_id[1] = 2;
638 config_->ts_layer_id[2] = 1;
639 config_->ts_layer_id[3] = 2;
640 } else {
641 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
642 }
643
644 if (external_ref_control_) {
645 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS;
646 if (num_temporal_layers_ > 1 && num_spatial_layers_ > 1 &&
647 codec_.mode == VideoCodecMode::kScreensharing) {
648 // External reference control for several temporal layers with different
649 // frame rates on spatial layers is not implemented yet.
650 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
651 }
652 }
653 ref_buf_.clear();
654
655 return InitAndSetControlSettings(inst);
656 }
657
NumberOfThreads(int width,int height,int number_of_cores)658 int VP9EncoderImpl::NumberOfThreads(int width,
659 int height,
660 int number_of_cores) {
661 // Keep the number of encoder threads equal to the possible number of column
662 // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS.
663 if (width * height >= 1280 * 720 && number_of_cores > 4) {
664 return 4;
665 } else if (width * height >= 640 * 360 && number_of_cores > 2) {
666 return 2;
667 } else {
668 // Use 2 threads for low res on ARM.
669 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
670 defined(WEBRTC_ANDROID)
671 if (width * height >= 320 * 180 && number_of_cores > 2) {
672 return 2;
673 }
674 #endif
675 // 1 thread less than VGA.
676 return 1;
677 }
678 }
679
InitAndSetControlSettings(const VideoCodec * inst)680 int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
681 // Set QP-min/max per spatial and temporal layer.
682 int tot_num_layers = num_spatial_layers_ * num_temporal_layers_;
683 for (int i = 0; i < tot_num_layers; ++i) {
684 svc_params_.max_quantizers[i] = config_->rc_max_quantizer;
685 svc_params_.min_quantizers[i] = config_->rc_min_quantizer;
686 }
687 config_->ss_number_layers = num_spatial_layers_;
688 if (ExplicitlyConfiguredSpatialLayers()) {
689 for (int i = 0; i < num_spatial_layers_; ++i) {
690 const auto& layer = codec_.spatialLayers[i];
691 RTC_CHECK_GT(layer.width, 0);
692 const int scale_factor = codec_.width / layer.width;
693 RTC_DCHECK_GT(scale_factor, 0);
694
695 // Ensure scaler factor is integer.
696 if (scale_factor * layer.width != codec_.width) {
697 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
698 }
699
700 // Ensure scale factor is the same in both dimensions.
701 if (scale_factor * layer.height != codec_.height) {
702 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
703 }
704
705 // Ensure scale factor is power of two.
706 const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0;
707 if (!is_pow_of_two) {
708 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
709 }
710
711 svc_params_.scaling_factor_num[i] = 1;
712 svc_params_.scaling_factor_den[i] = scale_factor;
713
714 RTC_DCHECK_GT(codec_.spatialLayers[i].maxFramerate, 0);
715 RTC_DCHECK_LE(codec_.spatialLayers[i].maxFramerate, codec_.maxFramerate);
716 if (i > 0) {
717 // Frame rate of high spatial layer is supposed to be equal or higher
718 // than frame rate of low spatial layer.
719 RTC_DCHECK_GE(codec_.spatialLayers[i].maxFramerate,
720 codec_.spatialLayers[i - 1].maxFramerate);
721 }
722 }
723 } else {
724 int scaling_factor_num = 256;
725 for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
726 // 1:2 scaling in each dimension.
727 svc_params_.scaling_factor_num[i] = scaling_factor_num;
728 svc_params_.scaling_factor_den[i] = 256;
729 }
730 }
731
732 SvcRateAllocator init_allocator(codec_);
733 current_bitrate_allocation_ =
734 init_allocator.Allocate(VideoBitrateAllocationParameters(
735 inst->startBitrate * 1000, inst->maxFramerate));
736 if (!SetSvcRates(current_bitrate_allocation_)) {
737 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
738 }
739
740 const vpx_codec_err_t rv = vpx_codec_enc_init(
741 encoder_, vpx_codec_vp9_cx(), config_,
742 config_->g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH);
743 if (rv != VPX_CODEC_OK) {
744 RTC_LOG(LS_ERROR) << "Init error: " << vpx_codec_err_to_string(rv);
745 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
746 }
747 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
748 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
749 rc_max_intra_target_);
750 vpx_codec_control(encoder_, VP9E_SET_AQ_MODE,
751 inst->VP9().adaptiveQpMode ? 3 : 0);
752
753 vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
754 vpx_codec_control(encoder_, VP9E_SET_SVC_GF_TEMPORAL_REF, 0);
755
756 if (is_svc_) {
757 vpx_codec_control(encoder_, VP9E_SET_SVC, 1);
758 vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_);
759 }
760
761 if (num_spatial_layers_ > 1) {
762 switch (inter_layer_pred_) {
763 case InterLayerPredMode::kOn:
764 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0);
765 break;
766 case InterLayerPredMode::kOff:
767 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1);
768 break;
769 case InterLayerPredMode::kOnKeyPic:
770 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2);
771 break;
772 default:
773 RTC_NOTREACHED();
774 }
775
776 memset(&svc_drop_frame_, 0, sizeof(svc_drop_frame_));
777 const bool reverse_constrained_drop_mode =
778 inter_layer_pred_ == InterLayerPredMode::kOn &&
779 codec_.mode == VideoCodecMode::kScreensharing &&
780 num_spatial_layers_ > 1;
781 if (reverse_constrained_drop_mode) {
782 // Screenshare dropping mode: drop a layer only together with all lower
783 // layers. This ensures that drops on lower layers won't reduce frame-rate
784 // for higher layers and reference structure is RTP-compatible.
785 svc_drop_frame_.framedrop_mode = CONSTRAINED_FROM_ABOVE_DROP;
786 svc_drop_frame_.max_consec_drop = 5;
787 for (size_t i = 0; i < num_spatial_layers_; ++i) {
788 svc_drop_frame_.framedrop_thresh[i] = config_->rc_dropframe_thresh;
789 }
790 // No buffering is needed because the highest layer is always present in
791 // all frames in CONSTRAINED_FROM_ABOVE drop mode.
792 layer_buffering_ = false;
793 } else {
794 // Configure encoder to drop entire superframe whenever it needs to drop
795 // a layer. This mode is preferred over per-layer dropping which causes
796 // quality flickering and is not compatible with RTP non-flexible mode.
797 svc_drop_frame_.framedrop_mode =
798 full_superframe_drop_ ? FULL_SUPERFRAME_DROP : CONSTRAINED_LAYER_DROP;
799 // Buffering is needed only for constrained layer drop, as it's not clear
800 // which frame is the last.
801 layer_buffering_ = !full_superframe_drop_;
802 svc_drop_frame_.max_consec_drop = std::numeric_limits<int>::max();
803 for (size_t i = 0; i < num_spatial_layers_; ++i) {
804 svc_drop_frame_.framedrop_thresh[i] = config_->rc_dropframe_thresh;
805 }
806 }
807 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER,
808 &svc_drop_frame_);
809 }
810
811 // Register callback for getting each spatial layer.
812 vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = {
813 VP9EncoderImpl::EncoderOutputCodedPacketCallback,
814 reinterpret_cast<void*>(this)};
815 vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK,
816 reinterpret_cast<void*>(&cbp));
817
818 // Control function to set the number of column tiles in encoding a frame, in
819 // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns.
820 // The number tile columns will be capped by the encoder based on image size
821 // (minimum width of tile column is 256 pixels, maximum is 4096).
822 vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1));
823
824 // Turn on row-based multithreading.
825 vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1);
826
827 #if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
828 !defined(ANDROID)
829 // Do not enable the denoiser on ARM since optimization is pending.
830 // Denoiser is on by default on other platforms.
831 vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY,
832 inst->VP9().denoisingOn ? 1 : 0);
833 #endif
834
835 if (codec_.mode == VideoCodecMode::kScreensharing) {
836 // Adjust internal parameters to screen content.
837 vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1);
838 }
839 // Enable encoder skip of static/low content blocks.
840 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
841 inited_ = true;
842 config_changed_ = true;
843 return WEBRTC_VIDEO_CODEC_OK;
844 }
845
MaxIntraTarget(uint32_t optimal_buffer_size)846 uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) {
847 // Set max to the optimal buffer level (normalized by target BR),
848 // and scaled by a scale_par.
849 // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps].
850 // This value is presented in percentage of perFrameBw:
851 // perFrameBw = targetBR[Kbps] * 1000 / framerate.
852 // The target in % is as follows:
853 float scale_par = 0.5;
854 uint32_t target_pct =
855 optimal_buffer_size * scale_par * codec_.maxFramerate / 10;
856 // Don't go below 3 times the per frame bandwidth.
857 const uint32_t min_intra_size = 300;
858 return (target_pct < min_intra_size) ? min_intra_size : target_pct;
859 }
860
Encode(const VideoFrame & input_image,const std::vector<VideoFrameType> * frame_types)861 int VP9EncoderImpl::Encode(const VideoFrame& input_image,
862 const std::vector<VideoFrameType>* frame_types) {
863 if (!inited_) {
864 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
865 }
866 if (encoded_complete_callback_ == nullptr) {
867 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
868 }
869 if (num_active_spatial_layers_ == 0) {
870 // All spatial layers are disabled, return without encoding anything.
871 return WEBRTC_VIDEO_CODEC_OK;
872 }
873
874 // We only support one stream at the moment.
875 if (frame_types && !frame_types->empty()) {
876 if ((*frame_types)[0] == VideoFrameType::kVideoFrameKey) {
877 force_key_frame_ = true;
878 }
879 }
880
881 if (pics_since_key_ + 1 ==
882 static_cast<size_t>(codec_.VP9()->keyFrameInterval)) {
883 force_key_frame_ = true;
884 }
885
886 vpx_svc_layer_id_t layer_id = {0};
887 if (!force_key_frame_) {
888 const size_t gof_idx = (pics_since_key_ + 1) % gof_.num_frames_in_gof;
889 layer_id.temporal_layer_id = gof_.temporal_idx[gof_idx];
890
891 if (VideoCodecMode::kScreensharing == codec_.mode) {
892 const uint32_t frame_timestamp_ms =
893 1000 * input_image.timestamp() / kVideoPayloadTypeFrequency;
894
895 // To ensure that several rate-limiters with different limits don't
896 // interfere, they must be queried in order of increasing limit.
897
898 bool use_steady_state_limiter =
899 variable_framerate_experiment_.enabled &&
900 input_image.update_rect().IsEmpty() &&
901 num_steady_state_frames_ >=
902 variable_framerate_experiment_.frames_before_steady_state;
903
904 // Need to check all frame limiters, even if lower layers are disabled,
905 // because variable frame-rate limiter should be checked after the first
906 // layer. It's easier to overwrite active layers after, then check all
907 // cases.
908 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
909 const float layer_fps =
910 framerate_controller_[layer_id.spatial_layer_id].GetTargetRate();
911 // Use steady state rate-limiter at the correct place.
912 if (use_steady_state_limiter &&
913 layer_fps > variable_framerate_experiment_.framerate_limit - 1e-9) {
914 if (variable_framerate_controller_.DropFrame(frame_timestamp_ms)) {
915 layer_id.spatial_layer_id = num_active_spatial_layers_;
916 }
917 // Break always: if rate limiter triggered frame drop, no need to
918 // continue; otherwise, the rate is less than the next limiters.
919 break;
920 }
921 if (framerate_controller_[sl_idx].DropFrame(frame_timestamp_ms)) {
922 ++layer_id.spatial_layer_id;
923 } else {
924 break;
925 }
926 }
927
928 if (use_steady_state_limiter &&
929 layer_id.spatial_layer_id < num_active_spatial_layers_) {
930 variable_framerate_controller_.AddFrame(frame_timestamp_ms);
931 }
932 }
933
934 if (force_all_active_layers_) {
935 layer_id.spatial_layer_id = first_active_layer_;
936 force_all_active_layers_ = false;
937 }
938
939 RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
940 if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
941 // Drop entire picture.
942 return WEBRTC_VIDEO_CODEC_OK;
943 }
944 }
945
946 // Need to set temporal layer id on ALL layers, even disabled ones.
947 // Otherwise libvpx might produce frames on a disabled layer:
948 // http://crbug.com/1051476
949 for (int sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
950 layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id;
951 }
952
953 if (layer_id.spatial_layer_id < first_active_layer_) {
954 layer_id.spatial_layer_id = first_active_layer_;
955 }
956
957 vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
958
959 if (num_spatial_layers_ > 1) {
960 // Update frame dropping settings as they may change on per-frame basis.
961 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER,
962 &svc_drop_frame_);
963 }
964
965 if (config_changed_) {
966 if (vpx_codec_enc_config_set(encoder_, config_)) {
967 return WEBRTC_VIDEO_CODEC_ERROR;
968 }
969 config_changed_ = false;
970 }
971
972 if (input_image.width() != codec_.width ||
973 input_image.height() != codec_.height) {
974 int ret = UpdateCodecFrameSize(input_image);
975 if (ret < 0) {
976 return ret;
977 }
978 }
979
980 RTC_DCHECK_EQ(input_image.width(), raw_->d_w);
981 RTC_DCHECK_EQ(input_image.height(), raw_->d_h);
982
983 // Set input image for use in the callback.
984 // This was necessary since you need some information from input_image.
985 // You can save only the necessary information (such as timestamp) instead of
986 // doing this.
987 input_image_ = &input_image;
988
989 // Keep reference to buffer until encode completes.
990 rtc::scoped_refptr<I420BufferInterface> i420_buffer;
991 const I010BufferInterface* i010_buffer;
992 rtc::scoped_refptr<const I010BufferInterface> i010_copy;
993 switch (profile_) {
994 case VP9Profile::kProfile0: {
995 i420_buffer = input_image.video_frame_buffer()->ToI420();
996 // Image in vpx_image_t format.
997 // Input image is const. VPX's raw image is not defined as const.
998 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
999 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
1000 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
1001 raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
1002 raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
1003 raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
1004 break;
1005 }
1006 case VP9Profile::kProfile1: {
1007 RTC_NOTREACHED();
1008 break;
1009 }
1010 case VP9Profile::kProfile2: {
1011 // We can inject kI010 frames directly for encode. All other formats
1012 // should be converted to it.
1013 switch (input_image.video_frame_buffer()->type()) {
1014 case VideoFrameBuffer::Type::kI010: {
1015 i010_buffer = input_image.video_frame_buffer()->GetI010();
1016 break;
1017 }
1018 default: {
1019 i010_copy =
1020 I010Buffer::Copy(*input_image.video_frame_buffer()->ToI420());
1021 i010_buffer = i010_copy.get();
1022 }
1023 }
1024 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(
1025 reinterpret_cast<const uint8_t*>(i010_buffer->DataY()));
1026 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(
1027 reinterpret_cast<const uint8_t*>(i010_buffer->DataU()));
1028 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(
1029 reinterpret_cast<const uint8_t*>(i010_buffer->DataV()));
1030 raw_->stride[VPX_PLANE_Y] = i010_buffer->StrideY() * 2;
1031 raw_->stride[VPX_PLANE_U] = i010_buffer->StrideU() * 2;
1032 raw_->stride[VPX_PLANE_V] = i010_buffer->StrideV() * 2;
1033 break;
1034 }
1035 }
1036
1037 vpx_enc_frame_flags_t flags = 0;
1038 if (force_key_frame_) {
1039 flags = VPX_EFLAG_FORCE_KF;
1040 }
1041
1042 if (external_ref_control_) {
1043 vpx_svc_ref_frame_config_t ref_config =
1044 SetReferences(force_key_frame_, layer_id.spatial_layer_id);
1045
1046 if (VideoCodecMode::kScreensharing == codec_.mode) {
1047 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
1048 ref_config.duration[sl_idx] = static_cast<int64_t>(
1049 90000 / (std::min(static_cast<float>(codec_.maxFramerate),
1050 framerate_controller_[sl_idx].GetTargetRate())));
1051 }
1052 }
1053
1054 vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &ref_config);
1055 }
1056
1057 first_frame_in_picture_ = true;
1058
1059 // TODO(ssilkin): Frame duration should be specified per spatial layer
1060 // since their frame rate can be different. For now calculate frame duration
1061 // based on target frame rate of the highest spatial layer, which frame rate
1062 // is supposed to be equal or higher than frame rate of low spatial layers.
1063 // Also, timestamp should represent actual time passed since previous frame
1064 // (not 'expected' time). Then rate controller can drain buffer more
1065 // accurately.
1066 RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_);
1067 float target_framerate_fps =
1068 (codec_.mode == VideoCodecMode::kScreensharing)
1069 ? std::min(static_cast<float>(codec_.maxFramerate),
1070 framerate_controller_[num_active_spatial_layers_ - 1]
1071 .GetTargetRate())
1072 : codec_.maxFramerate;
1073 uint32_t duration = static_cast<uint32_t>(90000 / target_framerate_fps);
1074 const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
1075 duration, flags, VPX_DL_REALTIME);
1076 if (rv != VPX_CODEC_OK) {
1077 RTC_LOG(LS_ERROR) << "Encoding error: " << vpx_codec_err_to_string(rv)
1078 << "\n"
1079 "Details: "
1080 << vpx_codec_error(encoder_) << "\n"
1081 << vpx_codec_error_detail(encoder_);
1082 return WEBRTC_VIDEO_CODEC_ERROR;
1083 }
1084 timestamp_ += duration;
1085
1086 if (layer_buffering_) {
1087 const bool end_of_picture = true;
1088 DeliverBufferedFrame(end_of_picture);
1089 }
1090
1091 return WEBRTC_VIDEO_CODEC_OK;
1092 }
1093
UpdateCodecFrameSize(const VideoFrame & input_image)1094 int VP9EncoderImpl::UpdateCodecFrameSize(
1095 const VideoFrame& input_image) {
1096 RTC_LOG(LS_INFO) << "Reconfiging VP from " <<
1097 codec_.width << "x" << codec_.height << " to " <<
1098 input_image.width() << "x" << input_image.height();
1099 // Preserve latest bitrate/framerate setting
1100 // TODO: Mozilla - see below, we need to save more state here.
1101 //uint32_t old_bitrate_kbit = config_->rc_target_bitrate;
1102 //uint32_t old_framerate = codec_.maxFramerate;
1103
1104 codec_.width = input_image.width();
1105 codec_.height = input_image.height();
1106
1107 vpx_img_free(raw_);
1108 raw_ = vpx_img_wrap(NULL, VPX_IMG_FMT_I420, codec_.width, codec_.height,
1109 1, NULL);
1110 // Update encoder context for new frame size.
1111 config_->g_w = codec_.width;
1112 config_->g_h = codec_.height;
1113
1114 // Determine number of threads based on the image size and #cores.
1115 config_->g_threads = NumberOfThreads(codec_.width, codec_.height,
1116 num_cores_);
1117 // Update the cpu_speed setting for resolution change.
1118 cpu_speed_ = GetCpuSpeed(codec_.width, codec_.height);
1119
1120 // NOTE: We would like to do this the same way vp8 does it
1121 // (with vpx_codec_enc_config_set()), but that causes asserts
1122 // in AQ 3 (cyclic); and in AQ 0 it works, but on a resize to smaller
1123 // than 1/2 x 1/2 original it asserts in convolve(). Given these
1124 // bugs in trying to do it the "right" way, we basically re-do
1125 // the initialization.
1126 vpx_codec_destroy(encoder_); // clean up old state
1127 int result = InitAndSetControlSettings(&codec_);
1128 if (result == WEBRTC_VIDEO_CODEC_OK) {
1129 // TODO: Mozilla rates have become much more complicated, we need to store
1130 // more state or find another way of doing this.
1131 //return SetRates(old_bitrate_kbit, old_framerate);
1132 RTC_CHECK(false);
1133 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1134 }
1135 return result;
1136 }
1137
PopulateCodecSpecific(CodecSpecificInfo * codec_specific,absl::optional<int> * spatial_idx,const vpx_codec_cx_pkt & pkt,uint32_t timestamp)1138 void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
1139 absl::optional<int>* spatial_idx,
1140 const vpx_codec_cx_pkt& pkt,
1141 uint32_t timestamp) {
1142 RTC_CHECK(codec_specific != nullptr);
1143 codec_specific->codecType = kVideoCodecVP9;
1144 CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9);
1145
1146 vp9_info->first_frame_in_picture = first_frame_in_picture_;
1147 vp9_info->flexible_mode = is_flexible_mode_;
1148
1149 if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
1150 pics_since_key_ = 0;
1151 } else if (first_frame_in_picture_) {
1152 ++pics_since_key_;
1153 }
1154
1155 vpx_svc_layer_id_t layer_id = {0};
1156 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1157
1158 // Can't have keyframe with non-zero temporal layer.
1159 RTC_DCHECK(pics_since_key_ != 0 || layer_id.temporal_layer_id == 0);
1160
1161 RTC_CHECK_GT(num_temporal_layers_, 0);
1162 RTC_CHECK_GT(num_active_spatial_layers_, 0);
1163 if (num_temporal_layers_ == 1) {
1164 RTC_CHECK_EQ(layer_id.temporal_layer_id, 0);
1165 vp9_info->temporal_idx = kNoTemporalIdx;
1166 } else {
1167 vp9_info->temporal_idx = layer_id.temporal_layer_id;
1168 }
1169 if (num_active_spatial_layers_ == 1) {
1170 RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
1171 *spatial_idx = absl::nullopt;
1172 } else {
1173 *spatial_idx = layer_id.spatial_layer_id;
1174 }
1175
1176 // TODO(asapersson): this info has to be obtained from the encoder.
1177 vp9_info->temporal_up_switch = false;
1178
1179 const bool is_key_pic = (pics_since_key_ == 0);
1180 const bool is_inter_layer_pred_allowed =
1181 (inter_layer_pred_ == InterLayerPredMode::kOn ||
1182 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic));
1183
1184 // Always set inter_layer_predicted to true on high layer frame if inter-layer
1185 // prediction (ILP) is allowed even if encoder didn't actually use it.
1186 // Setting inter_layer_predicted to false would allow receiver to decode high
1187 // layer frame without decoding low layer frame. If that would happen (e.g.
1188 // if low layer frame is lost) then receiver won't be able to decode next high
1189 // layer frame which uses ILP.
1190 vp9_info->inter_layer_predicted =
1191 first_frame_in_picture_ ? false : is_inter_layer_pred_allowed;
1192
1193 // Mark all low spatial layer frames as references (not just frames of
1194 // active low spatial layers) if inter-layer prediction is enabled since
1195 // these frames are indirect references of high spatial layer, which can
1196 // later be enabled without key frame.
1197 vp9_info->non_ref_for_inter_layer_pred =
1198 !is_inter_layer_pred_allowed ||
1199 layer_id.spatial_layer_id + 1 == num_spatial_layers_;
1200
1201 // Always populate this, so that the packetizer can properly set the marker
1202 // bit.
1203 vp9_info->num_spatial_layers = num_active_spatial_layers_;
1204 vp9_info->first_active_layer = first_active_layer_;
1205
1206 vp9_info->num_ref_pics = 0;
1207 FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted,
1208 vp9_info);
1209 if (vp9_info->flexible_mode) {
1210 vp9_info->gof_idx = kNoGofIdx;
1211 } else {
1212 vp9_info->gof_idx =
1213 static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof);
1214 vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx];
1215 RTC_DCHECK(vp9_info->num_ref_pics == gof_.num_ref_pics[vp9_info->gof_idx] ||
1216 vp9_info->num_ref_pics == 0);
1217 }
1218
1219 vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0);
1220
1221 // Write SS on key frame of independently coded spatial layers and on base
1222 // temporal/spatial layer frame if number of layers changed without issuing
1223 // of key picture (inter-layer prediction is enabled).
1224 const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted;
1225 if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 &&
1226 layer_id.spatial_layer_id == first_active_layer_)) {
1227 vp9_info->ss_data_available = true;
1228 vp9_info->spatial_layer_resolution_present = true;
1229 // Signal disabled layers.
1230 for (size_t i = 0; i < first_active_layer_; ++i) {
1231 vp9_info->width[i] = 0;
1232 vp9_info->height[i] = 0;
1233 }
1234 for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) {
1235 vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
1236 svc_params_.scaling_factor_den[i];
1237 vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
1238 svc_params_.scaling_factor_den[i];
1239 }
1240 if (vp9_info->flexible_mode) {
1241 vp9_info->gof.num_frames_in_gof = 0;
1242 } else {
1243 vp9_info->gof.CopyGofInfoVP9(gof_);
1244 }
1245
1246 ss_info_needed_ = false;
1247 } else {
1248 vp9_info->ss_data_available = false;
1249 }
1250
1251 first_frame_in_picture_ = false;
1252 }
1253
FillReferenceIndices(const vpx_codec_cx_pkt & pkt,const size_t pic_num,const bool inter_layer_predicted,CodecSpecificInfoVP9 * vp9_info)1254 void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
1255 const size_t pic_num,
1256 const bool inter_layer_predicted,
1257 CodecSpecificInfoVP9* vp9_info) {
1258 vpx_svc_layer_id_t layer_id = {0};
1259 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1260
1261 const bool is_key_frame =
1262 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1263
1264 std::vector<RefFrameBuffer> ref_buf_list;
1265
1266 if (is_svc_) {
1267 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1268 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
1269 int ref_buf_flags = 0;
1270
1271 if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) {
1272 const size_t fb_idx =
1273 enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id];
1274 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
1275 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1276 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1277 ref_buf_list.push_back(ref_buf_.at(fb_idx));
1278 ref_buf_flags |= 1 << fb_idx;
1279 }
1280 }
1281
1282 if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) {
1283 const size_t fb_idx =
1284 enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id];
1285 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
1286 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1287 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1288 ref_buf_list.push_back(ref_buf_.at(fb_idx));
1289 ref_buf_flags |= 1 << fb_idx;
1290 }
1291 }
1292
1293 if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) {
1294 const size_t fb_idx =
1295 enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id];
1296 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
1297 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1298 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1299 ref_buf_list.push_back(ref_buf_.at(fb_idx));
1300 ref_buf_flags |= 1 << fb_idx;
1301 }
1302 }
1303
1304 RTC_LOG(LS_VERBOSE) << "Frame " << pic_num << " sl "
1305 << layer_id.spatial_layer_id << " tl "
1306 << layer_id.temporal_layer_id << " refered buffers "
1307 << (ref_buf_flags & (1 << 0) ? 1 : 0)
1308 << (ref_buf_flags & (1 << 1) ? 1 : 0)
1309 << (ref_buf_flags & (1 << 2) ? 1 : 0)
1310 << (ref_buf_flags & (1 << 3) ? 1 : 0)
1311 << (ref_buf_flags & (1 << 4) ? 1 : 0)
1312 << (ref_buf_flags & (1 << 5) ? 1 : 0)
1313 << (ref_buf_flags & (1 << 6) ? 1 : 0)
1314 << (ref_buf_flags & (1 << 7) ? 1 : 0);
1315
1316 } else if (!is_key_frame) {
1317 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1318 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1319 // In non-SVC mode encoder doesn't provide reference list. Assume each frame
1320 // refers previous one, which is stored in buffer 0.
1321 ref_buf_list.push_back(ref_buf_.at(0));
1322 }
1323
1324 size_t max_ref_temporal_layer_id = 0;
1325
1326 std::vector<size_t> ref_pid_list;
1327
1328 vp9_info->num_ref_pics = 0;
1329 for (const RefFrameBuffer& ref_buf : ref_buf_list) {
1330 RTC_DCHECK_LE(ref_buf.pic_num, pic_num);
1331 if (ref_buf.pic_num < pic_num) {
1332 if (inter_layer_pred_ != InterLayerPredMode::kOn) {
1333 // RTP spec limits temporal prediction to the same spatial layer.
1334 // It is safe to ignore this requirement if inter-layer prediction is
1335 // enabled for all frames when all base frames are relayed to receiver.
1336 RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
1337 } else {
1338 RTC_DCHECK_LE(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
1339 }
1340 RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id);
1341
1342 // Encoder may reference several spatial layers on the same previous
1343 // frame in case if some spatial layers are skipped on the current frame.
1344 // We shouldn't put duplicate references as it may break some old
1345 // clients and isn't RTP compatible.
1346 if (std::find(ref_pid_list.begin(), ref_pid_list.end(),
1347 ref_buf.pic_num) != ref_pid_list.end()) {
1348 continue;
1349 }
1350 ref_pid_list.push_back(ref_buf.pic_num);
1351
1352 const size_t p_diff = pic_num - ref_buf.pic_num;
1353 RTC_DCHECK_LE(p_diff, 127UL);
1354
1355 vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff);
1356 ++vp9_info->num_ref_pics;
1357
1358 max_ref_temporal_layer_id =
1359 std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id);
1360 } else {
1361 RTC_DCHECK(inter_layer_predicted);
1362 // RTP spec only allows to use previous spatial layer for inter-layer
1363 // prediction.
1364 RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id);
1365 }
1366 }
1367
1368 vp9_info->temporal_up_switch =
1369 (max_ref_temporal_layer_id <
1370 static_cast<size_t>(layer_id.temporal_layer_id));
1371 }
1372
UpdateReferenceBuffers(const vpx_codec_cx_pkt & pkt,const size_t pic_num)1373 void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
1374 const size_t pic_num) {
1375 vpx_svc_layer_id_t layer_id = {0};
1376 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1377
1378 RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id,
1379 layer_id.temporal_layer_id);
1380
1381 if (is_svc_) {
1382 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1383 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
1384 const int update_buffer_slot =
1385 enc_layer_conf.update_buffer_slot[layer_id.spatial_layer_id];
1386
1387 for (size_t i = 0; i < kNumVp9Buffers; ++i) {
1388 if (update_buffer_slot & (1 << i)) {
1389 ref_buf_[i] = frame_buf;
1390 }
1391 }
1392
1393 RTC_LOG(LS_VERBOSE) << "Frame " << pic_num << " sl "
1394 << layer_id.spatial_layer_id << " tl "
1395 << layer_id.temporal_layer_id << " updated buffers "
1396 << (update_buffer_slot & (1 << 0) ? 1 : 0)
1397 << (update_buffer_slot & (1 << 1) ? 1 : 0)
1398 << (update_buffer_slot & (1 << 2) ? 1 : 0)
1399 << (update_buffer_slot & (1 << 3) ? 1 : 0)
1400 << (update_buffer_slot & (1 << 4) ? 1 : 0)
1401 << (update_buffer_slot & (1 << 5) ? 1 : 0)
1402 << (update_buffer_slot & (1 << 6) ? 1 : 0)
1403 << (update_buffer_slot & (1 << 7) ? 1 : 0);
1404 } else {
1405 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1406 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1407 // In non-svc mode encoder doesn't provide reference list. Assume each frame
1408 // is reference and stored in buffer 0.
1409 ref_buf_[0] = frame_buf;
1410 }
1411 }
1412
SetReferences(bool is_key_pic,size_t first_active_spatial_layer_id)1413 vpx_svc_ref_frame_config_t VP9EncoderImpl::SetReferences(
1414 bool is_key_pic,
1415 size_t first_active_spatial_layer_id) {
1416 // kRefBufIdx, kUpdBufIdx need to be updated to support longer GOFs.
1417 RTC_DCHECK_LE(gof_.num_frames_in_gof, 4);
1418
1419 vpx_svc_ref_frame_config_t ref_config;
1420 memset(&ref_config, 0, sizeof(ref_config));
1421
1422 const size_t num_temporal_refs = std::max(1, num_temporal_layers_ - 1);
1423 const bool is_inter_layer_pred_allowed =
1424 inter_layer_pred_ == InterLayerPredMode::kOn ||
1425 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic);
1426 absl::optional<int> last_updated_buf_idx;
1427
1428 // Put temporal reference to LAST and spatial reference to GOLDEN. Update
1429 // frame buffer (i.e. store encoded frame) if current frame is a temporal
1430 // reference (i.e. it belongs to a low temporal layer) or it is a spatial
1431 // reference. In later case, always store spatial reference in the last
1432 // reference frame buffer.
1433 // For the case of 3 temporal and 3 spatial layers we need 6 frame buffers
1434 // for temporal references plus 1 buffer for spatial reference. 7 buffers
1435 // in total.
1436
1437 for (size_t sl_idx = first_active_spatial_layer_id;
1438 sl_idx < num_active_spatial_layers_; ++sl_idx) {
1439 const size_t curr_pic_num = is_key_pic ? 0 : pics_since_key_ + 1;
1440 const size_t gof_idx = curr_pic_num % gof_.num_frames_in_gof;
1441
1442 if (!is_key_pic) {
1443 // Set up temporal reference.
1444 const int buf_idx = sl_idx * num_temporal_refs + kRefBufIdx[gof_idx];
1445
1446 // Last reference frame buffer is reserved for spatial reference. It is
1447 // not supposed to be used for temporal prediction.
1448 RTC_DCHECK_LT(buf_idx, kNumVp9Buffers - 1);
1449
1450 const int pid_diff = curr_pic_num - ref_buf_[buf_idx].pic_num;
1451 // Incorrect spatial layer may be in the buffer due to a key-frame.
1452 const bool same_spatial_layer =
1453 ref_buf_[buf_idx].spatial_layer_id == sl_idx;
1454 bool correct_pid = false;
1455 if (is_flexible_mode_) {
1456 correct_pid = pid_diff > 0 && pid_diff < kMaxAllowedPidDiff;
1457 } else {
1458 // Below code assumes single temporal referecence.
1459 RTC_DCHECK_EQ(gof_.num_ref_pics[gof_idx], 1);
1460 correct_pid = pid_diff == gof_.pid_diff[gof_idx][0];
1461 }
1462
1463 if (same_spatial_layer && correct_pid) {
1464 ref_config.lst_fb_idx[sl_idx] = buf_idx;
1465 ref_config.reference_last[sl_idx] = 1;
1466 } else {
1467 // This reference doesn't match with one specified by GOF. This can
1468 // only happen if spatial layer is enabled dynamically without key
1469 // frame. Spatial prediction is supposed to be enabled in this case.
1470 RTC_DCHECK(is_inter_layer_pred_allowed &&
1471 sl_idx > first_active_spatial_layer_id);
1472 }
1473 }
1474
1475 if (is_inter_layer_pred_allowed && sl_idx > first_active_spatial_layer_id) {
1476 // Set up spatial reference.
1477 RTC_DCHECK(last_updated_buf_idx);
1478 ref_config.gld_fb_idx[sl_idx] = *last_updated_buf_idx;
1479 ref_config.reference_golden[sl_idx] = 1;
1480 } else {
1481 RTC_DCHECK(ref_config.reference_last[sl_idx] != 0 ||
1482 sl_idx == first_active_spatial_layer_id ||
1483 inter_layer_pred_ == InterLayerPredMode::kOff);
1484 }
1485
1486 last_updated_buf_idx.reset();
1487
1488 if (gof_.temporal_idx[gof_idx] < num_temporal_layers_ - 1 ||
1489 num_temporal_layers_ == 1) {
1490 last_updated_buf_idx = sl_idx * num_temporal_refs + kUpdBufIdx[gof_idx];
1491
1492 // Ensure last frame buffer is not used for temporal prediction (it is
1493 // reserved for spatial reference).
1494 RTC_DCHECK_LT(*last_updated_buf_idx, kNumVp9Buffers - 1);
1495 } else if (is_inter_layer_pred_allowed) {
1496 last_updated_buf_idx = kNumVp9Buffers - 1;
1497 }
1498
1499 if (last_updated_buf_idx) {
1500 ref_config.update_buffer_slot[sl_idx] = 1 << *last_updated_buf_idx;
1501 }
1502 }
1503
1504 return ref_config;
1505 }
1506
GetEncodedLayerFrame(const vpx_codec_cx_pkt * pkt)1507 int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
1508 RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT);
1509
1510 if (pkt->data.frame.sz == 0) {
1511 // Ignore dropped frame.
1512 return WEBRTC_VIDEO_CODEC_OK;
1513 }
1514
1515 vpx_svc_layer_id_t layer_id = {0};
1516 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1517
1518 if (layer_buffering_) {
1519 // Deliver buffered low spatial layer frame.
1520 const bool end_of_picture = false;
1521 DeliverBufferedFrame(end_of_picture);
1522 }
1523
1524 // TODO(nisse): Introduce some buffer cache or buffer pool, to reduce
1525 // allocations and/or copy operations.
1526 encoded_image_.SetEncodedData(EncodedImageBuffer::Create(
1527 static_cast<const uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz));
1528
1529 const bool is_key_frame =
1530 (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1531 // Ensure encoder issued key frame on request.
1532 RTC_DCHECK(is_key_frame || !force_key_frame_);
1533
1534 // Check if encoded frame is a key frame.
1535 encoded_image_._frameType = VideoFrameType::kVideoFrameDelta;
1536 if (is_key_frame) {
1537 encoded_image_._frameType = VideoFrameType::kVideoFrameKey;
1538 force_key_frame_ = false;
1539 }
1540 RTC_DCHECK_LE(encoded_image_.size(), encoded_image_.capacity());
1541
1542 codec_specific_ = {};
1543 absl::optional<int> spatial_index;
1544 PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt,
1545 input_image_->timestamp());
1546 encoded_image_.SetSpatialIndex(spatial_index);
1547
1548 UpdateReferenceBuffers(*pkt, pics_since_key_);
1549
1550 TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_.size());
1551 encoded_image_.SetTimestamp(input_image_->timestamp());
1552 encoded_image_._encodedHeight =
1553 pkt->data.frame.height[layer_id.spatial_layer_id];
1554 encoded_image_._encodedWidth =
1555 pkt->data.frame.width[layer_id.spatial_layer_id];
1556 int qp = -1;
1557 vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
1558 encoded_image_.qp_ = qp;
1559
1560 if (!layer_buffering_) {
1561 const bool end_of_picture = encoded_image_.SpatialIndex().value_or(0) + 1 ==
1562 num_active_spatial_layers_;
1563 DeliverBufferedFrame(end_of_picture);
1564 }
1565
1566 return WEBRTC_VIDEO_CODEC_OK;
1567 }
1568
DeliverBufferedFrame(bool end_of_picture)1569 void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
1570 if (encoded_image_.size() > 0) {
1571 if (num_spatial_layers_ > 1) {
1572 // Restore frame dropping settings, as dropping may be temporary forbidden
1573 // due to dynamically enabled layers.
1574 for (size_t i = 0; i < num_spatial_layers_; ++i) {
1575 svc_drop_frame_.framedrop_thresh[i] = config_->rc_dropframe_thresh;
1576 }
1577 }
1578
1579 codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture;
1580
1581 encoded_complete_callback_->OnEncodedImage(encoded_image_,
1582 &codec_specific_);
1583
1584 if (codec_.mode == VideoCodecMode::kScreensharing) {
1585 const uint8_t spatial_idx = encoded_image_.SpatialIndex().value_or(0);
1586 const uint32_t frame_timestamp_ms =
1587 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
1588 framerate_controller_[spatial_idx].AddFrame(frame_timestamp_ms);
1589
1590 const size_t steady_state_size = SteadyStateSize(
1591 spatial_idx, codec_specific_.codecSpecific.VP9.temporal_idx);
1592
1593 // Only frames on spatial layers, which may be limited in a steady state
1594 // are considered for steady state detection.
1595 if (framerate_controller_[spatial_idx].GetTargetRate() >
1596 variable_framerate_experiment_.framerate_limit + 1e-9) {
1597 if (encoded_image_.qp_ <=
1598 variable_framerate_experiment_.steady_state_qp &&
1599 encoded_image_.size() <= steady_state_size) {
1600 ++num_steady_state_frames_;
1601 } else {
1602 num_steady_state_frames_ = 0;
1603 }
1604 }
1605 }
1606 encoded_image_.set_size(0);
1607 }
1608 }
1609
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)1610 int VP9EncoderImpl::RegisterEncodeCompleteCallback(
1611 EncodedImageCallback* callback) {
1612 encoded_complete_callback_ = callback;
1613 return WEBRTC_VIDEO_CODEC_OK;
1614 }
1615
GetEncoderInfo() const1616 VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
1617 EncoderInfo info;
1618 info.supports_native_handle = false;
1619 info.implementation_name = "libvpx";
1620 if (quality_scaler_experiment_.enabled) {
1621 info.scaling_settings = VideoEncoder::ScalingSettings(
1622 quality_scaler_experiment_.low_qp, quality_scaler_experiment_.high_qp);
1623 } else {
1624 info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
1625 }
1626 info.has_trusted_rate_controller = trusted_rate_controller_;
1627 info.is_hardware_accelerated = false;
1628 info.has_internal_source = false;
1629 if (inited_) {
1630 // Find the max configured fps of any active spatial layer.
1631 float max_fps = 0.0;
1632 for (size_t si = 0; si < num_spatial_layers_; ++si) {
1633 if (codec_.spatialLayers[si].active &&
1634 codec_.spatialLayers[si].maxFramerate > max_fps) {
1635 max_fps = codec_.spatialLayers[si].maxFramerate;
1636 }
1637 }
1638
1639 for (size_t si = 0; si < num_spatial_layers_; ++si) {
1640 info.fps_allocation[si].clear();
1641 if (!codec_.spatialLayers[si].active) {
1642 continue;
1643 }
1644
1645 // This spatial layer may already use a fraction of the total frame rate.
1646 const float sl_fps_fraction =
1647 codec_.spatialLayers[si].maxFramerate / max_fps;
1648 for (size_t ti = 0; ti < num_temporal_layers_; ++ti) {
1649 const uint32_t decimator =
1650 num_temporal_layers_ <= 1 ? 1 : config_->ts_rate_decimator[ti];
1651 RTC_DCHECK_GT(decimator, 0);
1652 info.fps_allocation[si].push_back(
1653 rtc::saturated_cast<uint8_t>(EncoderInfo::kMaxFramerateFraction *
1654 (sl_fps_fraction / decimator)));
1655 }
1656 }
1657 }
1658 return info;
1659 }
1660
SteadyStateSize(int sid,int tid)1661 size_t VP9EncoderImpl::SteadyStateSize(int sid, int tid) {
1662 const size_t bitrate_bps = current_bitrate_allocation_.GetBitrate(
1663 sid, tid == kNoTemporalIdx ? 0 : tid);
1664 const float fps = (codec_.mode == VideoCodecMode::kScreensharing)
1665 ? std::min(static_cast<float>(codec_.maxFramerate),
1666 framerate_controller_[sid].GetTargetRate())
1667 : codec_.maxFramerate;
1668 return static_cast<size_t>(
1669 bitrate_bps / (8 * fps) *
1670 (100 -
1671 variable_framerate_experiment_.steady_state_undershoot_percentage) /
1672 100 +
1673 0.5);
1674 }
1675
1676 // static
1677 VP9EncoderImpl::VariableFramerateExperiment
ParseVariableFramerateConfig(std::string group_name)1678 VP9EncoderImpl::ParseVariableFramerateConfig(std::string group_name) {
1679 FieldTrialFlag enabled = FieldTrialFlag("Enabled");
1680 FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
1681 FieldTrialParameter<int> qp("min_qp", 32);
1682 FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
1683 FieldTrialParameter<int> frames_before_steady_state(
1684 "frames_before_steady_state", 5);
1685 ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage,
1686 &frames_before_steady_state},
1687 field_trial::FindFullName(group_name));
1688 VariableFramerateExperiment config;
1689 config.enabled = enabled.Get();
1690 config.framerate_limit = framerate_limit.Get();
1691 config.steady_state_qp = qp.Get();
1692 config.steady_state_undershoot_percentage = undershoot_percentage.Get();
1693 config.frames_before_steady_state = frames_before_steady_state.Get();
1694
1695 return config;
1696 }
1697
1698 // static
1699 VP9EncoderImpl::QualityScalerExperiment
ParseQualityScalerConfig(std::string group_name)1700 VP9EncoderImpl::ParseQualityScalerConfig(std::string group_name) {
1701 FieldTrialFlag disabled = FieldTrialFlag("Disabled");
1702 FieldTrialParameter<int> low_qp("low_qp", kLowVp9QpThreshold);
1703 FieldTrialParameter<int> high_qp("hihg_qp", kHighVp9QpThreshold);
1704 ParseFieldTrial({&disabled, &low_qp, &high_qp},
1705 field_trial::FindFullName(group_name));
1706 QualityScalerExperiment config;
1707 config.enabled = !disabled.Get();
1708 RTC_LOG(LS_INFO) << "Webrtc quality scaler for vp9 is "
1709 << (config.enabled ? "enabled." : "disabled");
1710 config.low_qp = low_qp.Get();
1711 config.high_qp = high_qp.Get();
1712
1713 return config;
1714 }
1715
VP9DecoderImpl()1716 VP9DecoderImpl::VP9DecoderImpl()
1717 : decode_complete_callback_(nullptr),
1718 inited_(false),
1719 decoder_(nullptr),
1720 key_frame_required_(true) {}
1721
~VP9DecoderImpl()1722 VP9DecoderImpl::~VP9DecoderImpl() {
1723 inited_ = true; // in order to do the actual release
1724 Release();
1725 int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
1726 if (num_buffers_in_use > 0) {
1727 // The frame buffers are reference counted and frames are exposed after
1728 // decoding. There may be valid usage cases where previous frames are still
1729 // referenced after ~VP9DecoderImpl that is not a leak.
1730 RTC_LOG(LS_INFO) << num_buffers_in_use
1731 << " Vp9FrameBuffers are still "
1732 "referenced during ~VP9DecoderImpl.";
1733 }
1734 }
1735
InitDecode(const VideoCodec * inst,int number_of_cores)1736 int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
1737 int ret_val = Release();
1738 if (ret_val < 0) {
1739 return ret_val;
1740 }
1741
1742 if (decoder_ == nullptr) {
1743 decoder_ = new vpx_codec_ctx_t;
1744 }
1745 vpx_codec_dec_cfg_t cfg;
1746 memset(&cfg, 0, sizeof(cfg));
1747
1748 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
1749 // We focus on webrtc fuzzing here, not libvpx itself. Use single thread for
1750 // fuzzing, because:
1751 // - libvpx's VP9 single thread decoder is more fuzzer friendly. It detects
1752 // errors earlier than the multi-threads version.
1753 // - Make peak CPU usage under control (not depending on input)
1754 cfg.threads = 1;
1755 #else
1756 if (!inst) {
1757 // No config provided - don't know resolution to decode yet.
1758 // Set thread count to one in the meantime.
1759 cfg.threads = 1;
1760 } else {
1761 // We want to use multithreading when decoding high resolution videos. But
1762 // not too many in order to avoid overhead when many stream are decoded
1763 // concurrently.
1764 // Set 2 thread as target for 1280x720 pixel count, and then scale up
1765 // linearly from there - but cap at physical core count.
1766 // For common resolutions this results in:
1767 // 1 for 360p
1768 // 2 for 720p
1769 // 4 for 1080p
1770 // 8 for 1440p
1771 // 18 for 4K
1772 int num_threads =
1773 std::max(1, 2 * (inst->width * inst->height) / (1280 * 720));
1774 cfg.threads = std::min(number_of_cores, num_threads);
1775 current_codec_ = *inst;
1776 }
1777 #endif
1778
1779 num_cores_ = number_of_cores;
1780
1781 vpx_codec_flags_t flags = 0;
1782 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
1783 return WEBRTC_VIDEO_CODEC_MEMORY;
1784 }
1785
1786 if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
1787 return WEBRTC_VIDEO_CODEC_MEMORY;
1788 }
1789
1790 inited_ = true;
1791 // Always start with a complete key frame.
1792 key_frame_required_ = true;
1793 if (inst && inst->buffer_pool_size) {
1794 if (!frame_buffer_pool_.Resize(*inst->buffer_pool_size)) {
1795 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1796 }
1797 }
1798
1799 vpx_codec_err_t status =
1800 vpx_codec_control(decoder_, VP9D_SET_LOOP_FILTER_OPT, 1);
1801 if (status != VPX_CODEC_OK) {
1802 RTC_LOG(LS_ERROR) << "Failed to enable VP9D_SET_LOOP_FILTER_OPT. "
1803 << vpx_codec_error(decoder_);
1804 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1805 }
1806
1807 return WEBRTC_VIDEO_CODEC_OK;
1808 }
1809
Decode(const EncodedImage & input_image,bool missing_frames,int64_t)1810 int VP9DecoderImpl::Decode(const EncodedImage& input_image,
1811 bool missing_frames,
1812 int64_t /*render_time_ms*/) {
1813 if (!inited_) {
1814 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1815 }
1816 if (decode_complete_callback_ == nullptr) {
1817 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1818 }
1819
1820 if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
1821 absl::optional<vp9::FrameInfo> frame_info =
1822 vp9::ParseIntraFrameInfo(input_image.data(), input_image.size());
1823 if (frame_info) {
1824 if (frame_info->frame_width != current_codec_.width ||
1825 frame_info->frame_height != current_codec_.height) {
1826 // Resolution has changed, tear down and re-init a new decoder in
1827 // order to get correct sizing.
1828 Release();
1829 current_codec_.width = frame_info->frame_width;
1830 current_codec_.height = frame_info->frame_height;
1831 int reinit_status = InitDecode(¤t_codec_, num_cores_);
1832 if (reinit_status != WEBRTC_VIDEO_CODEC_OK) {
1833 RTC_LOG(LS_WARNING) << "Failed to re-init decoder.";
1834 return reinit_status;
1835 }
1836 }
1837 } else {
1838 RTC_LOG(LS_WARNING) << "Failed to parse VP9 header from key-frame.";
1839 }
1840 }
1841
1842 // Always start with a complete key frame.
1843 if (key_frame_required_) {
1844 if (input_image._frameType != VideoFrameType::kVideoFrameKey)
1845 return WEBRTC_VIDEO_CODEC_ERROR;
1846 // We have a key frame - is it complete?
1847 if (input_image._completeFrame) {
1848 key_frame_required_ = false;
1849 } else {
1850 return WEBRTC_VIDEO_CODEC_ERROR;
1851 }
1852 }
1853 vpx_codec_iter_t iter = nullptr;
1854 vpx_image_t* img;
1855 const uint8_t* buffer = input_image.data();
1856 if (input_image.size() == 0) {
1857 buffer = nullptr; // Triggers full frame concealment.
1858 }
1859 // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
1860 // In practice libvpx keeps a few (~3-4) buffers alive at a time.
1861 if (vpx_codec_decode(decoder_, buffer,
1862 static_cast<unsigned int>(input_image.size()), 0,
1863 VPX_DL_REALTIME)) {
1864 return WEBRTC_VIDEO_CODEC_ERROR;
1865 }
1866 // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
1867 // It may be released by libvpx during future vpx_codec_decode or
1868 // vpx_codec_destroy calls.
1869 img = vpx_codec_get_frame(decoder_, &iter);
1870 int qp;
1871 vpx_codec_err_t vpx_ret =
1872 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
1873 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
1874 int ret =
1875 ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace());
1876 if (ret != 0) {
1877 return ret;
1878 }
1879 return WEBRTC_VIDEO_CODEC_OK;
1880 }
1881
ReturnFrame(const vpx_image_t * img,uint32_t timestamp,int qp,const webrtc::ColorSpace * explicit_color_space)1882 int VP9DecoderImpl::ReturnFrame(
1883 const vpx_image_t* img,
1884 uint32_t timestamp,
1885 int qp,
1886 const webrtc::ColorSpace* explicit_color_space) {
1887 if (img == nullptr) {
1888 // Decoder OK and nullptr image => No show frame.
1889 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1890 }
1891
1892 // This buffer contains all of |img|'s image data, a reference counted
1893 // Vp9FrameBuffer. (libvpx is done with the buffers after a few
1894 // vpx_codec_decode calls or vpx_codec_destroy).
1895 Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
1896 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
1897
1898 // The buffer can be used directly by the VideoFrame (without copy) by
1899 // using a Wrapped*Buffer.
1900 rtc::scoped_refptr<VideoFrameBuffer> img_wrapped_buffer;
1901 switch (img->bit_depth) {
1902 case 8:
1903 if (img->fmt == VPX_IMG_FMT_I420) {
1904 img_wrapped_buffer = WrapI420Buffer(
1905 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1906 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1907 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1908 img->stride[VPX_PLANE_V],
1909 // WrappedI420Buffer's mechanism for allowing the release of its
1910 // frame buffer is through a callback function. This is where we
1911 // should release |img_buffer|.
1912 rtc::KeepRefUntilDone(img_buffer));
1913 } else if (img->fmt == VPX_IMG_FMT_I444) {
1914 img_wrapped_buffer = WrapI444Buffer(
1915 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1916 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1917 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1918 img->stride[VPX_PLANE_V],
1919 // WrappedI444Buffer's mechanism for allowing the release of its
1920 // frame buffer is through a callback function. This is where we
1921 // should release |img_buffer|.
1922 rtc::KeepRefUntilDone(img_buffer));
1923 } else {
1924 RTC_LOG(LS_ERROR)
1925 << "Unsupported pixel format produced by the decoder: "
1926 << static_cast<int>(img->fmt);
1927 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1928 }
1929 break;
1930 case 10:
1931 img_wrapped_buffer = WrapI010Buffer(
1932 img->d_w, img->d_h,
1933 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]),
1934 img->stride[VPX_PLANE_Y] / 2,
1935 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]),
1936 img->stride[VPX_PLANE_U] / 2,
1937 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]),
1938 img->stride[VPX_PLANE_V] / 2, rtc::KeepRefUntilDone(img_buffer));
1939 break;
1940 default:
1941 RTC_LOG(LS_ERROR) << "Unsupported bit depth produced by the decoder: "
1942 << img->bit_depth;
1943 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1944 }
1945
1946 auto builder = VideoFrame::Builder()
1947 .set_video_frame_buffer(img_wrapped_buffer)
1948 .set_timestamp_rtp(timestamp);
1949 if (explicit_color_space) {
1950 builder.set_color_space(*explicit_color_space);
1951 } else {
1952 builder.set_color_space(
1953 ExtractVP9ColorSpace(img->cs, img->range, img->bit_depth));
1954 }
1955 VideoFrame decoded_image = builder.build();
1956
1957 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
1958 return WEBRTC_VIDEO_CODEC_OK;
1959 }
1960
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)1961 int VP9DecoderImpl::RegisterDecodeCompleteCallback(
1962 DecodedImageCallback* callback) {
1963 decode_complete_callback_ = callback;
1964 return WEBRTC_VIDEO_CODEC_OK;
1965 }
1966
Release()1967 int VP9DecoderImpl::Release() {
1968 int ret_val = WEBRTC_VIDEO_CODEC_OK;
1969
1970 if (decoder_ != nullptr) {
1971 if (inited_) {
1972 // When a codec is destroyed libvpx will release any buffers of
1973 // |frame_buffer_pool_| it is currently using.
1974 if (vpx_codec_destroy(decoder_)) {
1975 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
1976 }
1977 }
1978 delete decoder_;
1979 decoder_ = nullptr;
1980 }
1981 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
1982 // still referenced externally are deleted once fully released, not returning
1983 // to the pool.
1984 frame_buffer_pool_.ClearPool();
1985 inited_ = false;
1986 return ret_val;
1987 }
1988
ImplementationName() const1989 const char* VP9DecoderImpl::ImplementationName() const {
1990 return "libvpx";
1991 }
1992
1993 } // namespace webrtc
1994
1995 #endif // RTC_ENABLE_VP9
1996