1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
12
13 #include <assert.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <cstdint>
18 #include <iterator>
19 #include <memory>
20 #include <string>
21 #include <utility>
22 #include <vector>
23
24 #include "api/scoped_refptr.h"
25 #include "api/video/video_content_type.h"
26 #include "api/video/video_frame_buffer.h"
27 #include "api/video/video_timing.h"
28 #include "api/video_codecs/vp8_temporal_layers.h"
29 #include "api/video_codecs/vp8_temporal_layers_factory.h"
30 #include "modules/video_coding/codecs/interface/common_constants.h"
31 #include "modules/video_coding/codecs/vp8/include/vp8.h"
32 #include "modules/video_coding/include/video_error_codes.h"
33 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
34 #include "modules/video_coding/utility/simulcast_utility.h"
35 #include "rtc_base/checks.h"
36 #include "rtc_base/experiments/experimental_screenshare_settings.h"
37 #include "rtc_base/experiments/field_trial_parser.h"
38 #include "rtc_base/experiments/field_trial_units.h"
39 #include "rtc_base/logging.h"
40 #include "rtc_base/trace_event.h"
41 #include "system_wrappers/include/field_trial.h"
42 #include "third_party/libyuv/include/libyuv/scale.h"
43 #include "vpx/vp8cx.h"
44
45 namespace webrtc {
46 namespace {
47 #if defined(WEBRTC_IOS)
48 const char kVP8IosMaxNumberOfThreadFieldTrial[] =
49 "WebRTC-VP8IosMaxNumberOfThread";
50 const char kVP8IosMaxNumberOfThreadFieldTrialParameter[] = "max_thread";
51 #endif
52
53 const char kVp8ForcePartitionResilience[] =
54 "WebRTC-VP8-ForcePartitionResilience";
55
56 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
57 // bitstream range of [0, 127] and not the user-level range of [0,63].
58 constexpr int kLowVp8QpThreshold = 29;
59 constexpr int kHighVp8QpThreshold = 95;
60
61 constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION;
62 constexpr uint32_t kVp832ByteAlign = 32u;
63
64 constexpr int kRtpTicksPerSecond = 90000;
65 constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000;
66
67 constexpr double kLowRateFactor = 1.0;
68 constexpr double kHighRateFactor = 2.0;
69
70 // VP8 denoiser states.
71 enum denoiserState : uint32_t {
72 kDenoiserOff,
73 kDenoiserOnYOnly,
74 kDenoiserOnYUV,
75 kDenoiserOnYUVAggressive,
76 // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch
77 // to kDenoiserOnYUVAggressive based on a computed noise metric.
78 kDenoiserOnAdaptive
79 };
80
81 // These settings correspond to the settings in vpx_codec_enc_cfg.
82 struct Vp8RateSettings {
83 uint32_t rc_undershoot_pct;
84 uint32_t rc_overshoot_pct;
85 uint32_t rc_buf_sz;
86 uint32_t rc_buf_optimal_sz;
87 uint32_t rc_dropframe_thresh;
88 };
89
90 // Greatest common divisior
GCD(int a,int b)91 int GCD(int a, int b) {
92 int c = a % b;
93 while (c != 0) {
94 a = b;
95 b = c;
96 c = a % b;
97 }
98 return b;
99 }
100
Interpolate(uint32_t low,uint32_t high,double bandwidth_headroom_factor)101 uint32_t Interpolate(uint32_t low,
102 uint32_t high,
103 double bandwidth_headroom_factor) {
104 RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor);
105 RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor);
106
107 // |factor| is between 0.0 and 1.0.
108 const double factor = bandwidth_headroom_factor - kLowRateFactor;
109
110 return static_cast<uint32_t>(((1.0 - factor) * low) + (factor * high) + 0.5);
111 }
112
GetRateSettings(double bandwidth_headroom_factor)113 Vp8RateSettings GetRateSettings(double bandwidth_headroom_factor) {
114 static const Vp8RateSettings low_settings{1000u, 0u, 100u, 30u, 40u};
115 static const Vp8RateSettings high_settings{100u, 15u, 1000u, 600u, 5u};
116
117 if (bandwidth_headroom_factor <= kLowRateFactor) {
118 return low_settings;
119 } else if (bandwidth_headroom_factor >= kHighRateFactor) {
120 return high_settings;
121 }
122
123 Vp8RateSettings settings;
124 settings.rc_undershoot_pct =
125 Interpolate(low_settings.rc_undershoot_pct,
126 high_settings.rc_undershoot_pct, bandwidth_headroom_factor);
127 settings.rc_overshoot_pct =
128 Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct,
129 bandwidth_headroom_factor);
130 settings.rc_buf_sz =
131 Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz,
132 bandwidth_headroom_factor);
133 settings.rc_buf_optimal_sz =
134 Interpolate(low_settings.rc_buf_optimal_sz,
135 high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor);
136 settings.rc_dropframe_thresh =
137 Interpolate(low_settings.rc_dropframe_thresh,
138 high_settings.rc_dropframe_thresh, bandwidth_headroom_factor);
139 return settings;
140 }
141
UpdateRateSettings(vpx_codec_enc_cfg_t * config,const Vp8RateSettings & new_settings)142 void UpdateRateSettings(vpx_codec_enc_cfg_t* config,
143 const Vp8RateSettings& new_settings) {
144 config->rc_undershoot_pct = new_settings.rc_undershoot_pct;
145 config->rc_overshoot_pct = new_settings.rc_overshoot_pct;
146 config->rc_buf_sz = new_settings.rc_buf_sz;
147 config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz;
148 config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh;
149 }
150
151 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxPeriodicity ==
152 VPX_TS_MAX_PERIODICITY,
153 "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the "
154 "constant in libvpx.");
155 static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers ==
156 VPX_TS_MAX_LAYERS,
157 "Vp8EncoderConfig::kMaxLayers must be kept in sync with the "
158 "constant in libvpx.");
159
160 // Allow a newer value to override a current value only if the new value
161 // is set.
162 template <typename T>
MaybeSetNewValue(const absl::optional<T> & new_value,absl::optional<T> * base_value)163 bool MaybeSetNewValue(const absl::optional<T>& new_value,
164 absl::optional<T>* base_value) {
165 if (new_value.has_value() && new_value != *base_value) {
166 *base_value = new_value;
167 return true;
168 } else {
169 return false;
170 }
171 }
172
173 // Adds configuration from |new_config| to |base_config|. Both configs consist
174 // of optionals, and only optionals which are set in |new_config| can have
175 // an effect. (That is, set values in |base_config| cannot be unset.)
176 // Returns |true| iff any changes were made to |base_config|.
MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig & new_config,Vp8EncoderConfig * base_config)177 bool MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig& new_config,
178 Vp8EncoderConfig* base_config) {
179 bool changes_made = false;
180 changes_made |= MaybeSetNewValue(new_config.temporal_layer_config,
181 &base_config->temporal_layer_config);
182 changes_made |= MaybeSetNewValue(new_config.rc_target_bitrate,
183 &base_config->rc_target_bitrate);
184 changes_made |= MaybeSetNewValue(new_config.rc_max_quantizer,
185 &base_config->rc_max_quantizer);
186 changes_made |= MaybeSetNewValue(new_config.g_error_resilient,
187 &base_config->g_error_resilient);
188 return changes_made;
189 }
190
ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig & encoder_config,vpx_codec_enc_cfg_t * vpx_config)191 void ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig& encoder_config,
192 vpx_codec_enc_cfg_t* vpx_config) {
193 if (encoder_config.temporal_layer_config.has_value()) {
194 const Vp8EncoderConfig::TemporalLayerConfig& ts_config =
195 encoder_config.temporal_layer_config.value();
196 vpx_config->ts_number_layers = ts_config.ts_number_layers;
197 std::copy(ts_config.ts_target_bitrate.begin(),
198 ts_config.ts_target_bitrate.end(),
199 std::begin(vpx_config->ts_target_bitrate));
200 std::copy(ts_config.ts_rate_decimator.begin(),
201 ts_config.ts_rate_decimator.end(),
202 std::begin(vpx_config->ts_rate_decimator));
203 vpx_config->ts_periodicity = ts_config.ts_periodicity;
204 std::copy(ts_config.ts_layer_id.begin(), ts_config.ts_layer_id.end(),
205 std::begin(vpx_config->ts_layer_id));
206 } else {
207 vpx_config->ts_number_layers = 1;
208 vpx_config->ts_rate_decimator[0] = 1;
209 vpx_config->ts_periodicity = 1;
210 vpx_config->ts_layer_id[0] = 0;
211 }
212
213 if (encoder_config.rc_target_bitrate.has_value()) {
214 vpx_config->rc_target_bitrate = encoder_config.rc_target_bitrate.value();
215 }
216
217 if (encoder_config.rc_max_quantizer.has_value()) {
218 vpx_config->rc_max_quantizer = encoder_config.rc_max_quantizer.value();
219 }
220
221 if (encoder_config.g_error_resilient.has_value()) {
222 vpx_config->g_error_resilient = encoder_config.g_error_resilient.value();
223 }
224 }
225
226 } // namespace
227
Create()228 std::unique_ptr<VideoEncoder> VP8Encoder::Create() {
229 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
230 VP8Encoder::Settings());
231 }
232
Create(VP8Encoder::Settings settings)233 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
234 VP8Encoder::Settings settings) {
235 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
236 std::move(settings));
237 }
238
Create(std::unique_ptr<Vp8FrameBufferControllerFactory> frame_buffer_controller_factory)239 std::unique_ptr<VideoEncoder> VP8Encoder::Create(
240 std::unique_ptr<Vp8FrameBufferControllerFactory>
241 frame_buffer_controller_factory) {
242 VP8Encoder::Settings settings;
243 settings.frame_buffer_controller_factory =
244 std::move(frame_buffer_controller_factory);
245 return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::CreateEncoder(),
246 std::move(settings));
247 }
248
EncodeFlags(const Vp8FrameConfig & references)249 vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags(
250 const Vp8FrameConfig& references) {
251 RTC_DCHECK(!references.drop_frame);
252
253 vpx_enc_frame_flags_t flags = 0;
254
255 if ((references.last_buffer_flags &
256 Vp8FrameConfig::BufferFlags::kReference) == 0)
257 flags |= VP8_EFLAG_NO_REF_LAST;
258 if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
259 0)
260 flags |= VP8_EFLAG_NO_UPD_LAST;
261 if ((references.golden_buffer_flags &
262 Vp8FrameConfig::BufferFlags::kReference) == 0)
263 flags |= VP8_EFLAG_NO_REF_GF;
264 if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
265 0)
266 flags |= VP8_EFLAG_NO_UPD_GF;
267 if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) ==
268 0)
269 flags |= VP8_EFLAG_NO_REF_ARF;
270 if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0)
271 flags |= VP8_EFLAG_NO_UPD_ARF;
272 if (references.freeze_entropy)
273 flags |= VP8_EFLAG_NO_UPD_ENTROPY;
274
275 return flags;
276 }
277
LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,VP8Encoder::Settings settings)278 LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
279 VP8Encoder::Settings settings)
280 : libvpx_(std::move(interface)),
281 experimental_cpu_speed_config_arm_(CpuSpeedExperiment::GetConfigs()),
282 rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
283 screenshare_max_qp_(
284 ExperimentalScreenshareSettings::ParseFromFieldTrials().MaxQp()),
285 frame_buffer_controller_factory_(
286 std::move(settings.frame_buffer_controller_factory)),
287 resolution_bitrate_limits_(std::move(settings.resolution_bitrate_limits)),
288 key_frame_request_(kMaxSimulcastStreams, false),
289 variable_framerate_experiment_(ParseVariableFramerateConfig(
290 "WebRTC-VP8VariableFramerateScreenshare")),
291 framerate_controller_(variable_framerate_experiment_.framerate_limit) {
292 // TODO(eladalon/ilnik): These reservations might be wasting memory.
293 // InitEncode() is resizing to the actual size, which might be smaller.
294 raw_images_.reserve(kMaxSimulcastStreams);
295 encoded_images_.reserve(kMaxSimulcastStreams);
296 send_stream_.reserve(kMaxSimulcastStreams);
297 cpu_speed_.assign(kMaxSimulcastStreams, cpu_speed_default_);
298 encoders_.reserve(kMaxSimulcastStreams);
299 vpx_configs_.reserve(kMaxSimulcastStreams);
300 config_overrides_.reserve(kMaxSimulcastStreams);
301 downsampling_factors_.reserve(kMaxSimulcastStreams);
302 }
303
~LibvpxVp8Encoder()304 LibvpxVp8Encoder::~LibvpxVp8Encoder() {
305 Release();
306 }
307
Release()308 int LibvpxVp8Encoder::Release() {
309 int ret_val = WEBRTC_VIDEO_CODEC_OK;
310
311 encoded_images_.clear();
312
313 if (inited_) {
314 for (auto it = encoders_.rbegin(); it != encoders_.rend(); ++it) {
315 if (libvpx_->codec_destroy(&*it)) {
316 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
317 }
318 }
319 }
320 encoders_.clear();
321
322 vpx_configs_.clear();
323 config_overrides_.clear();
324 send_stream_.clear();
325 cpu_speed_.clear();
326
327 for (auto it = raw_images_.rbegin(); it != raw_images_.rend(); ++it) {
328 libvpx_->img_free(&*it);
329 }
330 raw_images_.clear();
331
332 frame_buffer_controller_.reset();
333 inited_ = false;
334 return ret_val;
335 }
336
SetRates(const RateControlParameters & parameters)337 void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) {
338 if (!inited_) {
339 RTC_LOG(LS_WARNING) << "SetRates() while not initialize";
340 return;
341 }
342
343 if (encoders_[0].err) {
344 RTC_LOG(LS_WARNING) << "Encoder in error state.";
345 return;
346 }
347
348 if (parameters.framerate_fps < 1.0) {
349 RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= 1.0): "
350 << parameters.framerate_fps;
351 return;
352 }
353
354 if (parameters.bitrate.get_sum_bps() == 0) {
355 // Encoder paused, turn off all encoding.
356 const int num_streams = static_cast<size_t>(encoders_.size());
357 for (int i = 0; i < num_streams; ++i)
358 SetStreamState(false, i);
359 return;
360 }
361
362 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
363
364 if (encoders_.size() > 1) {
365 // If we have more than 1 stream, reduce the qp_max for the low resolution
366 // stream if frame rate is not too low. The trade-off with lower qp_max is
367 // possibly more dropped frames, so we only do this if the frame rate is
368 // above some threshold (base temporal layer is down to 1/4 for 3 layers).
369 // We may want to condition this on bitrate later.
370 if (rate_control_settings_.Vp8BoostBaseLayerQuality() &&
371 parameters.framerate_fps > 20.0) {
372 vpx_configs_[encoders_.size() - 1].rc_max_quantizer = 45;
373 } else {
374 // Go back to default value set in InitEncode.
375 vpx_configs_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
376 }
377 }
378
379 for (size_t i = 0; i < encoders_.size(); ++i) {
380 const size_t stream_idx = encoders_.size() - 1 - i;
381
382 unsigned int target_bitrate_kbps =
383 parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
384
385 bool send_stream = target_bitrate_kbps > 0;
386 if (send_stream || encoders_.size() > 1)
387 SetStreamState(send_stream, stream_idx);
388
389 vpx_configs_[i].rc_target_bitrate = target_bitrate_kbps;
390 if (send_stream) {
391 frame_buffer_controller_->OnRatesUpdated(
392 stream_idx, parameters.bitrate.GetTemporalLayerAllocation(stream_idx),
393 static_cast<int>(parameters.framerate_fps + 0.5));
394 }
395
396 UpdateVpxConfiguration(stream_idx);
397
398 if (rate_control_settings_.Vp8DynamicRateSettings()) {
399 // Tweak rate control settings based on available network headroom.
400 UpdateRateSettings(
401 &vpx_configs_[i],
402 GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
403 parameters.bitrate.get_sum_bps()));
404 }
405
406 vpx_codec_err_t err =
407 libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]);
408 if (err != VPX_CODEC_OK) {
409 RTC_LOG(LS_WARNING) << "Error configuring codec, error code: " << err;
410 }
411 }
412 }
413
OnPacketLossRateUpdate(float packet_loss_rate)414 void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) {
415 // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
416 if (frame_buffer_controller_) {
417 frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate);
418 }
419 }
420
OnRttUpdate(int64_t rtt_ms)421 void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) {
422 // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
423 if (frame_buffer_controller_) {
424 frame_buffer_controller_->OnRttUpdate(rtt_ms);
425 }
426 }
427
OnLossNotification(const LossNotification & loss_notification)428 void LibvpxVp8Encoder::OnLossNotification(
429 const LossNotification& loss_notification) {
430 if (frame_buffer_controller_) {
431 frame_buffer_controller_->OnLossNotification(loss_notification);
432 }
433 }
434
SetStreamState(bool send_stream,int stream_idx)435 void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) {
436 if (send_stream && !send_stream_[stream_idx]) {
437 // Need a key frame if we have not sent this stream before.
438 key_frame_request_[stream_idx] = true;
439 }
440 send_stream_[stream_idx] = send_stream;
441 }
442
SetFecControllerOverride(FecControllerOverride * fec_controller_override)443 void LibvpxVp8Encoder::SetFecControllerOverride(
444 FecControllerOverride* fec_controller_override) {
445 // TODO(bugs.webrtc.org/10769): Update downstream and remove ability to
446 // pass nullptr.
447 // RTC_DCHECK(fec_controller_override);
448 RTC_DCHECK(!fec_controller_override_);
449 fec_controller_override_ = fec_controller_override;
450 }
451
452 // TODO(eladalon): s/inst/codec_settings/g.
InitEncode(const VideoCodec * inst,const VideoEncoder::Settings & settings)453 int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst,
454 const VideoEncoder::Settings& settings) {
455 if (inst == NULL) {
456 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
457 }
458 if (inst->maxFramerate < 1) {
459 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
460 }
461 // allow zero to represent an unspecified maxBitRate
462 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
463 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
464 }
465 if (inst->width < 1 || inst->height < 1) {
466 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
467 }
468 if (settings.number_of_cores < 1) {
469 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
470 }
471
472 num_active_streams_ = 0;
473 for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) {
474 if (inst->simulcastStream[i].active) {
475 ++num_active_streams_;
476 }
477 }
478 if (inst->numberOfSimulcastStreams == 0 && inst->active) {
479 num_active_streams_ = 1;
480 }
481
482 if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) {
483 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
484 }
485
486 int retVal = Release();
487 if (retVal < 0) {
488 return retVal;
489 }
490
491 int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
492 if (number_of_streams > 1 &&
493 !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
494 return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
495 }
496
497 RTC_DCHECK(!frame_buffer_controller_);
498 if (frame_buffer_controller_factory_) {
499 frame_buffer_controller_ = frame_buffer_controller_factory_->Create(
500 *inst, settings, fec_controller_override_);
501 } else {
502 Vp8TemporalLayersFactory factory;
503 frame_buffer_controller_ =
504 factory.Create(*inst, settings, fec_controller_override_);
505 }
506 RTC_DCHECK(frame_buffer_controller_);
507
508 number_of_cores_ = settings.number_of_cores;
509 timestamp_ = 0;
510 codec_ = *inst;
511
512 // Code expects simulcastStream resolutions to be correct, make sure they are
513 // filled even when there are no simulcast layers.
514 if (codec_.numberOfSimulcastStreams == 0) {
515 codec_.simulcastStream[0].width = codec_.width;
516 codec_.simulcastStream[0].height = codec_.height;
517 }
518
519 encoded_images_.resize(number_of_streams);
520 encoders_.resize(number_of_streams);
521 vpx_configs_.resize(number_of_streams);
522 config_overrides_.resize(number_of_streams);
523 downsampling_factors_.resize(number_of_streams);
524 raw_images_.resize(number_of_streams);
525 send_stream_.resize(number_of_streams);
526 send_stream_[0] = true; // For non-simulcast case.
527 cpu_speed_.resize(number_of_streams);
528 std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
529
530 int idx = number_of_streams - 1;
531 for (int i = 0; i < (number_of_streams - 1); ++i, --idx) {
532 int gcd = GCD(inst->simulcastStream[idx].width,
533 inst->simulcastStream[idx - 1].width);
534 downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd;
535 downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd;
536 send_stream_[i] = false;
537 }
538 if (number_of_streams > 1) {
539 send_stream_[number_of_streams - 1] = false;
540 downsampling_factors_[number_of_streams - 1].num = 1;
541 downsampling_factors_[number_of_streams - 1].den = 1;
542 }
543 for (int i = 0; i < number_of_streams; ++i) {
544 encoded_images_[i]._completeFrame = true;
545 }
546 // populate encoder configuration with default values
547 if (libvpx_->codec_enc_config_default(vpx_codec_vp8_cx(), &vpx_configs_[0],
548 0)) {
549 return WEBRTC_VIDEO_CODEC_ERROR;
550 }
551 // setting the time base of the codec
552 vpx_configs_[0].g_timebase.num = 1;
553 vpx_configs_[0].g_timebase.den = kRtpTicksPerSecond;
554 vpx_configs_[0].g_lag_in_frames = 0; // 0- no frame lagging
555
556 // Set the error resilience mode for temporal layers (but not simulcast).
557 vpx_configs_[0].g_error_resilient =
558 (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)
559 ? VPX_ERROR_RESILIENT_DEFAULT
560 : 0;
561
562 // Override the error resilience mode if this is not simulcast, but we are
563 // using temporal layers.
564 if (field_trial::IsEnabled(kVp8ForcePartitionResilience) &&
565 (number_of_streams == 1) &&
566 (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)) {
567 RTC_LOG(LS_INFO) << "Overriding g_error_resilient from "
568 << vpx_configs_[0].g_error_resilient << " to "
569 << VPX_ERROR_RESILIENT_PARTITIONS;
570 vpx_configs_[0].g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS;
571 }
572
573 // rate control settings
574 vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(0);
575 vpx_configs_[0].rc_end_usage = VPX_CBR;
576 vpx_configs_[0].g_pass = VPX_RC_ONE_PASS;
577 // Handle resizing outside of libvpx.
578 vpx_configs_[0].rc_resize_allowed = 0;
579 vpx_configs_[0].rc_min_quantizer =
580 codec_.mode == VideoCodecMode::kScreensharing ? 12 : 2;
581 if (inst->qpMax >= vpx_configs_[0].rc_min_quantizer) {
582 qp_max_ = inst->qpMax;
583 }
584 if (rate_control_settings_.LibvpxVp8QpMax()) {
585 qp_max_ = std::max(rate_control_settings_.LibvpxVp8QpMax().value(),
586 static_cast<int>(vpx_configs_[0].rc_min_quantizer));
587 }
588 if (codec_.mode == VideoCodecMode::kScreensharing && screenshare_max_qp_) {
589 qp_max_ = *screenshare_max_qp_;
590 }
591 vpx_configs_[0].rc_max_quantizer = qp_max_;
592 vpx_configs_[0].rc_undershoot_pct = 100;
593 vpx_configs_[0].rc_overshoot_pct = 15;
594 vpx_configs_[0].rc_buf_initial_sz = 500;
595 vpx_configs_[0].rc_buf_optimal_sz = 600;
596 vpx_configs_[0].rc_buf_sz = 1000;
597
598 // Set the maximum target size of any key-frame.
599 rc_max_intra_target_ = MaxIntraTarget(vpx_configs_[0].rc_buf_optimal_sz);
600
601 if (inst->VP8().keyFrameInterval > 0) {
602 vpx_configs_[0].kf_mode = VPX_KF_AUTO;
603 vpx_configs_[0].kf_max_dist = inst->VP8().keyFrameInterval;
604 } else {
605 vpx_configs_[0].kf_mode = VPX_KF_DISABLED;
606 }
607
608 // Allow the user to set the complexity for the base stream.
609 switch (inst->VP8().complexity) {
610 case VideoCodecComplexity::kComplexityHigh:
611 cpu_speed_[0] = -5;
612 break;
613 case VideoCodecComplexity::kComplexityHigher:
614 cpu_speed_[0] = -4;
615 break;
616 case VideoCodecComplexity::kComplexityMax:
617 cpu_speed_[0] = -3;
618 break;
619 default:
620 cpu_speed_[0] = -6;
621 break;
622 }
623 cpu_speed_default_ = cpu_speed_[0];
624 // Set encoding complexity (cpu_speed) based on resolution and/or platform.
625 cpu_speed_[0] = GetCpuSpeed(inst->width, inst->height);
626 for (int i = 1; i < number_of_streams; ++i) {
627 cpu_speed_[i] =
628 GetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width,
629 inst->simulcastStream[number_of_streams - 1 - i].height);
630 }
631 vpx_configs_[0].g_w = inst->width;
632 vpx_configs_[0].g_h = inst->height;
633
634 // Determine number of threads based on the image size and #cores.
635 // TODO(fbarchard): Consider number of Simulcast layers.
636 vpx_configs_[0].g_threads = NumberOfThreads(
637 vpx_configs_[0].g_w, vpx_configs_[0].g_h, settings.number_of_cores);
638
639 // Creating a wrapper to the image - setting image data to NULL.
640 // Actual pointer will be set in encode. Setting align to 1, as it
641 // is meaningless (no memory allocation is done here).
642 libvpx_->img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width,
643 inst->height, 1, NULL);
644
645 // Note the order we use is different from webm, we have lowest resolution
646 // at position 0 and they have highest resolution at position 0.
647 const size_t stream_idx_cfg_0 = encoders_.size() - 1;
648 SimulcastRateAllocator init_allocator(codec_);
649 VideoBitrateAllocation allocation =
650 init_allocator.Allocate(VideoBitrateAllocationParameters(
651 inst->startBitrate * 1000, inst->maxFramerate));
652 std::vector<uint32_t> stream_bitrates;
653 for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
654 uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000;
655 stream_bitrates.push_back(bitrate);
656 }
657
658 vpx_configs_[0].rc_target_bitrate = stream_bitrates[stream_idx_cfg_0];
659 if (stream_bitrates[stream_idx_cfg_0] > 0) {
660 uint32_t maxFramerate =
661 inst->simulcastStream[stream_idx_cfg_0].maxFramerate;
662 if (!maxFramerate) {
663 maxFramerate = inst->maxFramerate;
664 }
665
666 frame_buffer_controller_->OnRatesUpdated(
667 stream_idx_cfg_0,
668 allocation.GetTemporalLayerAllocation(stream_idx_cfg_0), maxFramerate);
669 }
670 frame_buffer_controller_->SetQpLimits(stream_idx_cfg_0,
671 vpx_configs_[0].rc_min_quantizer,
672 vpx_configs_[0].rc_max_quantizer);
673 UpdateVpxConfiguration(stream_idx_cfg_0);
674 vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx_cfg_0);
675
676 for (size_t i = 1; i < encoders_.size(); ++i) {
677 const size_t stream_idx = encoders_.size() - 1 - i;
678 memcpy(&vpx_configs_[i], &vpx_configs_[0], sizeof(vpx_configs_[0]));
679
680 vpx_configs_[i].g_w = inst->simulcastStream[stream_idx].width;
681 vpx_configs_[i].g_h = inst->simulcastStream[stream_idx].height;
682
683 // Use 1 thread for lower resolutions.
684 vpx_configs_[i].g_threads = 1;
685
686 vpx_configs_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx);
687
688 // Setting alignment to 32 - as that ensures at least 16 for all
689 // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
690 // the y plane, but only half of it to the u and v planes.
691 libvpx_->img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
692 inst->simulcastStream[stream_idx].width,
693 inst->simulcastStream[stream_idx].height,
694 kVp832ByteAlign);
695 SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
696 vpx_configs_[i].rc_target_bitrate = stream_bitrates[stream_idx];
697 if (stream_bitrates[stream_idx] > 0) {
698 uint32_t maxFramerate = inst->simulcastStream[stream_idx].maxFramerate;
699 if (!maxFramerate) {
700 maxFramerate = inst->maxFramerate;
701 }
702 frame_buffer_controller_->OnRatesUpdated(
703 stream_idx, allocation.GetTemporalLayerAllocation(stream_idx),
704 maxFramerate);
705 }
706 frame_buffer_controller_->SetQpLimits(stream_idx,
707 vpx_configs_[i].rc_min_quantizer,
708 vpx_configs_[i].rc_max_quantizer);
709 UpdateVpxConfiguration(stream_idx);
710 }
711
712 return InitAndSetControlSettings();
713 }
714
GetCpuSpeed(int width,int height)715 int LibvpxVp8Encoder::GetCpuSpeed(int width, int height) {
716 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
717 defined(WEBRTC_ANDROID)
718 // On mobile platform, use a lower speed setting for lower resolutions for
719 // CPUs with 4 or more cores.
720 RTC_DCHECK_GT(number_of_cores_, 0);
721 if (number_of_cores_ <= 3)
722 return -12;
723
724 if (experimental_cpu_speed_config_arm_) {
725 return CpuSpeedExperiment::GetValue(width * height,
726 *experimental_cpu_speed_config_arm_);
727 }
728
729 if (width * height <= 352 * 288)
730 return -8;
731 else if (width * height <= 640 * 480)
732 return -10;
733 else
734 return -12;
735 #else
736 // For non-ARM, increase encoding complexity (i.e., use lower speed setting)
737 // if resolution is below CIF. Otherwise, keep the default/user setting
738 // (|cpu_speed_default_|) set on InitEncode via VP8().complexity.
739 if (width * height < 352 * 288)
740 return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_;
741 else
742 return cpu_speed_default_;
743 #endif
744 }
745
NumberOfThreads(int width,int height,int cpus)746 int LibvpxVp8Encoder::NumberOfThreads(int width, int height, int cpus) {
747 #if defined(WEBRTC_ANDROID)
748 if (width * height >= 320 * 180) {
749 if (cpus >= 4) {
750 // 3 threads for CPUs with 4 and more cores since most of times only 4
751 // cores will be active.
752 return 3;
753 } else if (cpus == 3 || cpus == 2) {
754 return 2;
755 } else {
756 return 1;
757 }
758 }
759 return 1;
760 #else
761 #if defined(WEBRTC_IOS)
762 std::string trial_string =
763 field_trial::FindFullName(kVP8IosMaxNumberOfThreadFieldTrial);
764 FieldTrialParameter<int> max_thread_number(
765 kVP8IosMaxNumberOfThreadFieldTrialParameter, 0);
766 ParseFieldTrial({&max_thread_number}, trial_string);
767 if (max_thread_number.Get() > 0) {
768 if (width * height < 320 * 180) {
769 return 1; // Use single thread for small screens
770 }
771 // thread number must be less than or equal to the number of CPUs.
772 return std::min(cpus, max_thread_number.Get());
773 }
774 #endif // defined(WEBRTC_IOS)
775 if (width * height >= 1920 * 1080 && cpus > 8) {
776 return 8; // 8 threads for 1080p on high perf machines.
777 } else if (width * height > 1280 * 960 && cpus >= 6) {
778 // 3 threads for 1080p.
779 return 3;
780 } else if (width * height > 640 * 480 && cpus >= 3) {
781 // Default 2 threads for qHD/HD, but allow 3 if core count is high enough,
782 // as this will allow more margin for high-core/low clock machines or if
783 // not built with highest optimization.
784 if (cpus >= 6) {
785 return 3;
786 }
787 return 2;
788 } else {
789 // 1 thread for VGA or less.
790 return 1;
791 }
792 #endif
793 }
794
InitAndSetControlSettings()795 int LibvpxVp8Encoder::InitAndSetControlSettings() {
796 vpx_codec_flags_t flags = 0;
797 flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
798
799 if (encoders_.size() > 1) {
800 int error = libvpx_->codec_enc_init_multi(
801 &encoders_[0], vpx_codec_vp8_cx(), &vpx_configs_[0], encoders_.size(),
802 flags, &downsampling_factors_[0]);
803 if (error) {
804 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
805 }
806 } else {
807 if (libvpx_->codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(),
808 &vpx_configs_[0], flags)) {
809 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
810 }
811 }
812 // Enable denoising for the highest resolution stream, and for
813 // the second highest resolution if we are doing more than 2
814 // spatial layers/streams.
815 // TODO(holmer): Investigate possibility of adding a libvpx API
816 // for getting the denoised frame from the encoder and using that
817 // when encoding lower resolution streams. Would it work with the
818 // multi-res encoding feature?
819 denoiserState denoiser_state = kDenoiserOnYOnly;
820 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
821 defined(WEBRTC_ANDROID)
822 denoiser_state = kDenoiserOnYOnly;
823 #else
824 denoiser_state = kDenoiserOnAdaptive;
825 #endif
826 libvpx_->codec_control(
827 &encoders_[0], VP8E_SET_NOISE_SENSITIVITY,
828 codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
829 if (encoders_.size() > 2) {
830 libvpx_->codec_control(
831 &encoders_[1], VP8E_SET_NOISE_SENSITIVITY,
832 codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
833 }
834 for (size_t i = 0; i < encoders_.size(); ++i) {
835 // Allow more screen content to be detected as static.
836 libvpx_->codec_control(
837 &(encoders_[i]), VP8E_SET_STATIC_THRESHOLD,
838 codec_.mode == VideoCodecMode::kScreensharing ? 100u : 1u);
839 libvpx_->codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
840 libvpx_->codec_control(
841 &(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
842 static_cast<vp8e_token_partitions>(kTokenPartitions));
843 libvpx_->codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
844 rc_max_intra_target_);
845 // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
846 // rate control (drop frames on large target bitrate overshoot)
847 libvpx_->codec_control(
848 &(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
849 codec_.mode == VideoCodecMode::kScreensharing ? 2u : 0u);
850 }
851 inited_ = true;
852 return WEBRTC_VIDEO_CODEC_OK;
853 }
854
MaxIntraTarget(uint32_t optimalBuffersize)855 uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) {
856 // Set max to the optimal buffer level (normalized by target BR),
857 // and scaled by a scalePar.
858 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
859 // This values is presented in percentage of perFrameBw:
860 // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
861 // The target in % is as follows:
862
863 float scalePar = 0.5;
864 uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
865
866 // Don't go below 3 times the per frame bandwidth.
867 const uint32_t minIntraTh = 300;
868 return (targetPct < minIntraTh) ? minIntraTh : targetPct;
869 }
870
FrameDropThreshold(size_t spatial_idx) const871 uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const {
872 bool enable_frame_dropping = codec_.VP8().frameDroppingOn;
873 // If temporal layers are used, they get to override the frame dropping
874 // setting, as eg. ScreenshareLayers does not work as intended with frame
875 // dropping on and DefaultTemporalLayers will have performance issues with
876 // frame dropping off.
877 RTC_DCHECK(frame_buffer_controller_);
878 RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount());
879 enable_frame_dropping =
880 frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx);
881 return enable_frame_dropping ? 30 : 0;
882 }
883
SteadyStateSize(int sid,int tid)884 size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) {
885 const int encoder_id = encoders_.size() - 1 - sid;
886 size_t bitrate_bps;
887 float fps;
888 if (SimulcastUtility::IsConferenceModeScreenshare(codec_) ||
889 vpx_configs_[encoder_id].ts_number_layers <= 1) {
890 // In conference screenshare there's no defined per temporal layer bitrate
891 // and framerate.
892 bitrate_bps = vpx_configs_[encoder_id].rc_target_bitrate * 1000;
893 fps = codec_.maxFramerate;
894 } else {
895 bitrate_bps = vpx_configs_[encoder_id].ts_target_bitrate[tid] * 1000;
896 fps = codec_.maxFramerate /
897 fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid], 1.0);
898 if (tid > 0) {
899 // Layer bitrate and fps are counted as a partial sums.
900 bitrate_bps -= vpx_configs_[encoder_id].ts_target_bitrate[tid - 1] * 1000;
901 fps = codec_.maxFramerate /
902 fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid - 1], 1.0);
903 }
904 }
905
906 if (fps < 1e-9)
907 return 0;
908 return static_cast<size_t>(
909 bitrate_bps / (8 * fps) *
910 (100 -
911 variable_framerate_experiment_.steady_state_undershoot_percentage) /
912 100 +
913 0.5);
914 }
915
UpdateVpxConfiguration(size_t stream_index)916 bool LibvpxVp8Encoder::UpdateVpxConfiguration(size_t stream_index) {
917 RTC_DCHECK(frame_buffer_controller_);
918
919 const size_t config_index = vpx_configs_.size() - 1 - stream_index;
920
921 RTC_DCHECK_LT(config_index, config_overrides_.size());
922 Vp8EncoderConfig* config = &config_overrides_[config_index];
923
924 const Vp8EncoderConfig new_config =
925 frame_buffer_controller_->UpdateConfiguration(stream_index);
926
927 if (new_config.reset_previous_configuration_overrides) {
928 *config = new_config;
929 return true;
930 }
931
932 const bool changes_made = MaybeExtendVp8EncoderConfig(new_config, config);
933
934 // Note that overrides must be applied even if they haven't changed.
935 RTC_DCHECK_LT(config_index, vpx_configs_.size());
936 vpx_codec_enc_cfg_t* vpx_config = &vpx_configs_[config_index];
937 ApplyVp8EncoderConfigToVpxConfig(*config, vpx_config);
938
939 return changes_made;
940 }
941
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)942 int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
943 const std::vector<VideoFrameType>* frame_types) {
944 RTC_DCHECK_EQ(frame.width(), codec_.width);
945 RTC_DCHECK_EQ(frame.height(), codec_.height);
946
947 if (!inited_)
948 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
949 if (encoded_complete_callback_ == NULL)
950 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
951
952 bool key_frame_requested = false;
953 for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
954 ++i) {
955 if (key_frame_request_[i] && send_stream_[i]) {
956 key_frame_requested = true;
957 break;
958 }
959 }
960 if (!key_frame_requested && frame_types) {
961 for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
962 ++i) {
963 if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey &&
964 send_stream_[i]) {
965 key_frame_requested = true;
966 break;
967 }
968 }
969 }
970
971 if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 &&
972 !key_frame_requested) {
973 if (variable_framerate_experiment_.enabled &&
974 framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) {
975 return WEBRTC_VIDEO_CODEC_OK;
976 }
977 framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs);
978 }
979
980 bool send_key_frame = key_frame_requested;
981 bool drop_frame = false;
982 bool retransmission_allowed = true;
983 Vp8FrameConfig tl_configs[kMaxSimulcastStreams];
984 for (size_t i = 0; i < encoders_.size(); ++i) {
985 tl_configs[i] =
986 frame_buffer_controller_->NextFrameConfig(i, frame.timestamp());
987 send_key_frame |= tl_configs[i].IntraFrame();
988 drop_frame |= tl_configs[i].drop_frame;
989 RTC_DCHECK(i == 0 ||
990 retransmission_allowed == tl_configs[i].retransmission_allowed);
991 retransmission_allowed = tl_configs[i].retransmission_allowed;
992 }
993
994 if (drop_frame && !send_key_frame) {
995 return WEBRTC_VIDEO_CODEC_OK;
996 }
997
998 vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
999 for (size_t i = 0; i < encoders_.size(); ++i) {
1000 flags[i] = send_key_frame ? VPX_EFLAG_FORCE_KF : EncodeFlags(tl_configs[i]);
1001 }
1002
1003 rtc::scoped_refptr<I420BufferInterface> input_image =
1004 frame.video_frame_buffer()->ToI420();
1005 // Since we are extracting raw pointers from |input_image| to
1006 // |raw_images_[0]|, the resolution of these frames must match.
1007 RTC_DCHECK_EQ(input_image->width(), raw_images_[0].d_w);
1008 RTC_DCHECK_EQ(input_image->height(), raw_images_[0].d_h);
1009
1010 // Image in vpx_image_t format.
1011 // Input image is const. VP8's raw image is not defined as const.
1012 raw_images_[0].planes[VPX_PLANE_Y] =
1013 const_cast<uint8_t*>(input_image->DataY());
1014 raw_images_[0].planes[VPX_PLANE_U] =
1015 const_cast<uint8_t*>(input_image->DataU());
1016 raw_images_[0].planes[VPX_PLANE_V] =
1017 const_cast<uint8_t*>(input_image->DataV());
1018
1019 raw_images_[0].stride[VPX_PLANE_Y] = input_image->StrideY();
1020 raw_images_[0].stride[VPX_PLANE_U] = input_image->StrideU();
1021 raw_images_[0].stride[VPX_PLANE_V] = input_image->StrideV();
1022
1023 struct CleanUpOnExit {
1024 explicit CleanUpOnExit(vpx_image_t& raw_image) : raw_image_(raw_image) {}
1025 ~CleanUpOnExit() {
1026 raw_image_.planes[VPX_PLANE_Y] = nullptr;
1027 raw_image_.planes[VPX_PLANE_U] = nullptr;
1028 raw_image_.planes[VPX_PLANE_V] = nullptr;
1029 }
1030 vpx_image_t& raw_image_;
1031 } clean_up_on_exit(raw_images_[0]);
1032
1033 for (size_t i = 1; i < encoders_.size(); ++i) {
1034 // Scale the image down a number of times by downsampling factor
1035 libyuv::I420Scale(
1036 raw_images_[i - 1].planes[VPX_PLANE_Y],
1037 raw_images_[i - 1].stride[VPX_PLANE_Y],
1038 raw_images_[i - 1].planes[VPX_PLANE_U],
1039 raw_images_[i - 1].stride[VPX_PLANE_U],
1040 raw_images_[i - 1].planes[VPX_PLANE_V],
1041 raw_images_[i - 1].stride[VPX_PLANE_V], raw_images_[i - 1].d_w,
1042 raw_images_[i - 1].d_h, raw_images_[i].planes[VPX_PLANE_Y],
1043 raw_images_[i].stride[VPX_PLANE_Y], raw_images_[i].planes[VPX_PLANE_U],
1044 raw_images_[i].stride[VPX_PLANE_U], raw_images_[i].planes[VPX_PLANE_V],
1045 raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w,
1046 raw_images_[i].d_h, libyuv::kFilterBilinear);
1047 }
1048
1049 if (send_key_frame) {
1050 // Adapt the size of the key frame when in screenshare with 1 temporal
1051 // layer.
1052 if (encoders_.size() == 1 &&
1053 codec_.mode == VideoCodecMode::kScreensharing &&
1054 codec_.VP8()->numberOfTemporalLayers <= 1) {
1055 const uint32_t forceKeyFrameIntraTh = 100;
1056 libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1057 forceKeyFrameIntraTh);
1058 }
1059
1060 std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
1061 }
1062
1063 // Set the encoder frame flags and temporal layer_id for each spatial stream.
1064 // Note that streams are defined starting from lowest resolution at
1065 // position 0 to highest resolution at position |encoders_.size() - 1|,
1066 // whereas |encoder_| is from highest to lowest resolution.
1067 for (size_t i = 0; i < encoders_.size(); ++i) {
1068 const size_t stream_idx = encoders_.size() - 1 - i;
1069
1070 if (UpdateVpxConfiguration(stream_idx)) {
1071 if (libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]))
1072 return WEBRTC_VIDEO_CODEC_ERROR;
1073 }
1074
1075 libvpx_->codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS,
1076 static_cast<int>(flags[stream_idx]));
1077 libvpx_->codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
1078 tl_configs[i].encoder_layer_id);
1079 }
1080 // TODO(holmer): Ideally the duration should be the timestamp diff of this
1081 // frame and the next frame to be encoded, which we don't have. Instead we
1082 // would like to use the duration of the previous frame. Unfortunately the
1083 // rate control seems to be off with that setup. Using the average input
1084 // frame rate to calculate an average duration for now.
1085 assert(codec_.maxFramerate > 0);
1086 uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate;
1087
1088 int error = WEBRTC_VIDEO_CODEC_OK;
1089 int num_tries = 0;
1090 // If the first try returns WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT
1091 // the frame must be reencoded with the same parameters again because
1092 // target bitrate is exceeded and encoder state has been reset.
1093 while (num_tries == 0 ||
1094 (num_tries == 1 &&
1095 error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) {
1096 ++num_tries;
1097 // Note we must pass 0 for |flags| field in encode call below since they are
1098 // set above in |libvpx_interface_->vpx_codec_control_| function for each
1099 // encoder/spatial layer.
1100 error = libvpx_->codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
1101 duration, 0, VPX_DL_REALTIME);
1102 // Reset specific intra frame thresholds, following the key frame.
1103 if (send_key_frame) {
1104 libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
1105 rc_max_intra_target_);
1106 }
1107 if (error)
1108 return WEBRTC_VIDEO_CODEC_ERROR;
1109 // Examines frame timestamps only.
1110 error = GetEncodedPartitions(frame, retransmission_allowed);
1111 }
1112 // TODO(sprang): Shouldn't we use the frame timestamp instead?
1113 timestamp_ += duration;
1114 return error;
1115 }
1116
PopulateCodecSpecific(CodecSpecificInfo * codec_specific,const vpx_codec_cx_pkt_t & pkt,int stream_idx,int encoder_idx,uint32_t timestamp)1117 void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
1118 const vpx_codec_cx_pkt_t& pkt,
1119 int stream_idx,
1120 int encoder_idx,
1121 uint32_t timestamp) {
1122 assert(codec_specific != NULL);
1123 codec_specific->codecType = kVideoCodecVP8;
1124 codec_specific->codecSpecific.VP8.keyIdx =
1125 kNoKeyIdx; // TODO(hlundin) populate this
1126 codec_specific->codecSpecific.VP8.nonReference =
1127 (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
1128
1129 int qp = 0;
1130 vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
1131 frame_buffer_controller_->OnEncodeDone(
1132 stream_idx, timestamp, encoded_images_[encoder_idx].size(),
1133 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific);
1134 }
1135
GetEncodedPartitions(const VideoFrame & input_image,bool retransmission_allowed)1136 int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image,
1137 bool retransmission_allowed) {
1138 int stream_idx = static_cast<int>(encoders_.size()) - 1;
1139 int result = WEBRTC_VIDEO_CODEC_OK;
1140 for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
1141 ++encoder_idx, --stream_idx) {
1142 vpx_codec_iter_t iter = NULL;
1143 encoded_images_[encoder_idx].set_size(0);
1144 encoded_images_[encoder_idx]._frameType = VideoFrameType::kVideoFrameDelta;
1145 CodecSpecificInfo codec_specific;
1146 const vpx_codec_cx_pkt_t* pkt = NULL;
1147
1148 size_t encoded_size = 0;
1149 while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1150 NULL) {
1151 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
1152 encoded_size += pkt->data.frame.sz;
1153 }
1154 }
1155
1156 // TODO(nisse): Introduce some buffer cache or buffer pool, to reduce
1157 // allocations and/or copy operations.
1158 auto buffer = EncodedImageBuffer::Create(encoded_size);
1159
1160 iter = NULL;
1161 size_t encoded_pos = 0;
1162 while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
1163 NULL) {
1164 switch (pkt->kind) {
1165 case VPX_CODEC_CX_FRAME_PKT: {
1166 RTC_CHECK_LE(encoded_pos + pkt->data.frame.sz, buffer->size());
1167 memcpy(&buffer->data()[encoded_pos], pkt->data.frame.buf,
1168 pkt->data.frame.sz);
1169 encoded_pos += pkt->data.frame.sz;
1170 break;
1171 }
1172 default:
1173 break;
1174 }
1175 // End of frame
1176 if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
1177 // check if encoded frame is a key frame
1178 if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
1179 encoded_images_[encoder_idx]._frameType =
1180 VideoFrameType::kVideoFrameKey;
1181 }
1182 encoded_images_[encoder_idx].SetEncodedData(buffer);
1183 encoded_images_[encoder_idx].set_size(encoded_pos);
1184 encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
1185 PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx,
1186 input_image.timestamp());
1187 break;
1188 }
1189 }
1190 encoded_images_[encoder_idx].SetTimestamp(input_image.timestamp());
1191 encoded_images_[encoder_idx].SetRetransmissionAllowed(
1192 retransmission_allowed);
1193
1194 if (send_stream_[stream_idx]) {
1195 if (encoded_images_[encoder_idx].size() > 0) {
1196 TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
1197 encoded_images_[encoder_idx].size());
1198 encoded_images_[encoder_idx]._encodedHeight =
1199 codec_.simulcastStream[stream_idx].height;
1200 encoded_images_[encoder_idx]._encodedWidth =
1201 codec_.simulcastStream[stream_idx].width;
1202 int qp_128 = -1;
1203 libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER,
1204 &qp_128);
1205 encoded_images_[encoder_idx].qp_ = qp_128;
1206 encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
1207 &codec_specific, nullptr);
1208 const size_t steady_state_size = SteadyStateSize(
1209 stream_idx, codec_specific.codecSpecific.VP8.temporalIdx);
1210 if (qp_128 > variable_framerate_experiment_.steady_state_qp ||
1211 encoded_images_[encoder_idx].size() > steady_state_size) {
1212 num_steady_state_frames_ = 0;
1213 } else {
1214 ++num_steady_state_frames_;
1215 }
1216 } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping(
1217 stream_idx)) {
1218 result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
1219 if (encoded_images_[encoder_idx].size() == 0) {
1220 // Dropped frame that will be re-encoded.
1221 frame_buffer_controller_->OnFrameDropped(stream_idx,
1222 input_image.timestamp());
1223 }
1224 }
1225 }
1226 }
1227 return result;
1228 }
1229
GetEncoderInfo() const1230 VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
1231 EncoderInfo info;
1232 info.supports_native_handle = false;
1233 info.implementation_name = "libvpx";
1234 info.has_trusted_rate_controller =
1235 rate_control_settings_.LibvpxVp8TrustedRateController();
1236 info.is_hardware_accelerated = false;
1237 info.has_internal_source = false;
1238 info.supports_simulcast = true;
1239 if (!resolution_bitrate_limits_.empty()) {
1240 info.resolution_bitrate_limits = resolution_bitrate_limits_;
1241 }
1242
1243 const bool enable_scaling =
1244 num_active_streams_ == 1 &&
1245 (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) &&
1246 codec_.VP8().automaticResizeOn;
1247
1248 info.scaling_settings = enable_scaling
1249 ? VideoEncoder::ScalingSettings(
1250 kLowVp8QpThreshold, kHighVp8QpThreshold)
1251 : VideoEncoder::ScalingSettings::kOff;
1252 if (rate_control_settings_.LibvpxVp8MinPixels()) {
1253 info.scaling_settings.min_pixels_per_frame =
1254 rate_control_settings_.LibvpxVp8MinPixels().value();
1255 }
1256
1257 if (inited_) {
1258 // |encoder_idx| is libvpx index where 0 is highest resolution.
1259 // |si| is simulcast index, where 0 is lowest resolution.
1260 for (size_t si = 0, encoder_idx = encoders_.size() - 1;
1261 si < encoders_.size(); ++si, --encoder_idx) {
1262 info.fps_allocation[si].clear();
1263 if ((codec_.numberOfSimulcastStreams > si &&
1264 !codec_.simulcastStream[si].active) ||
1265 (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
1266 // No defined frame rate fractions if not active or if using
1267 // ScreenshareLayers, leave vector empty and continue;
1268 continue;
1269 }
1270 if (vpx_configs_[encoder_idx].ts_number_layers <= 1) {
1271 info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
1272 } else {
1273 for (size_t ti = 0; ti < vpx_configs_[encoder_idx].ts_number_layers;
1274 ++ti) {
1275 RTC_DCHECK_GT(vpx_configs_[encoder_idx].ts_rate_decimator[ti], 0);
1276 info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
1277 EncoderInfo::kMaxFramerateFraction /
1278 vpx_configs_[encoder_idx].ts_rate_decimator[ti] +
1279 0.5));
1280 }
1281 }
1282 }
1283 }
1284
1285 return info;
1286 }
1287
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)1288 int LibvpxVp8Encoder::RegisterEncodeCompleteCallback(
1289 EncodedImageCallback* callback) {
1290 encoded_complete_callback_ = callback;
1291 return WEBRTC_VIDEO_CODEC_OK;
1292 }
1293
1294 // static
1295 LibvpxVp8Encoder::VariableFramerateExperiment
ParseVariableFramerateConfig(std::string group_name)1296 LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) {
1297 FieldTrialFlag enabled = FieldTrialFlag("Enabled");
1298 FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
1299 FieldTrialParameter<int> qp("min_qp", 15);
1300 FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
1301 ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage},
1302 field_trial::FindFullName(group_name));
1303 VariableFramerateExperiment config;
1304 config.enabled = enabled.Get();
1305 config.framerate_limit = framerate_limit.Get();
1306 config.steady_state_qp = qp.Get();
1307 config.steady_state_undershoot_percentage = undershoot_percentage.Get();
1308
1309 return config;
1310 }
1311
1312 } // namespace webrtc
1313