1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
11
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <memory>
16 #include <utility>
17 #include <vector>
18
19 #include "absl/algorithm/container.h"
20 #include "absl/base/macros.h"
21 #include "absl/types/optional.h"
22 #include "api/scoped_refptr.h"
23 #include "api/video/encoded_image.h"
24 #include "api/video/i420_buffer.h"
25 #include "api/video/video_frame.h"
26 #include "api/video_codecs/video_codec.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "modules/video_coding/include/video_codec_interface.h"
29 #include "modules/video_coding/include/video_error_codes.h"
30 #include "modules/video_coding/svc/create_scalability_structure.h"
31 #include "modules/video_coding/svc/scalable_video_controller.h"
32 #include "modules/video_coding/svc/scalable_video_controller_no_layering.h"
33 #include "rtc_base/checks.h"
34 #include "rtc_base/logging.h"
35 #include "third_party/libaom/source/libaom/aom/aom_codec.h"
36 #include "third_party/libaom/source/libaom/aom/aom_encoder.h"
37 #include "third_party/libaom/source/libaom/aom/aomcx.h"
38
39 namespace webrtc {
40 namespace {
41
42 // Encoder configuration parameters
43 constexpr int kQpMin = 10;
44 constexpr int kUsageProfile = AOM_USAGE_REALTIME;
45 constexpr int kMinQindex = 145; // Min qindex threshold for QP scaling.
46 constexpr int kMaxQindex = 205; // Max qindex threshold for QP scaling.
47 constexpr int kBitDepth = 8;
48 constexpr int kLagInFrames = 0; // No look ahead.
49 constexpr int kRtpTicksPerSecond = 90000;
50 constexpr float kMinimumFrameRate = 1.0;
51
52 // Only positive speeds, range for real-time coding currently is: 6 - 8.
53 // Lower means slower/better quality, higher means fastest/lower quality.
GetCpuSpeed(int width,int height,int number_of_cores)54 int GetCpuSpeed(int width, int height, int number_of_cores) {
55 // For smaller resolutions, use lower speed setting (get some coding gain at
56 // the cost of increased encoding complexity).
57 if (number_of_cores > 4 && width * height < 320 * 180)
58 return 6;
59 else if (width * height >= 1280 * 720)
60 return 9;
61 else if (width * height >= 640 * 360)
62 return 8;
63 else
64 return 7;
65 }
66
GetSuperblockSize(int width,int height,int threads)67 aom_superblock_size_t GetSuperblockSize(int width, int height, int threads) {
68 int resolution = width * height;
69 if (threads >= 4 && resolution >= 960 * 540 && resolution < 1920 * 1080)
70 return AOM_SUPERBLOCK_SIZE_64X64;
71 else
72 return AOM_SUPERBLOCK_SIZE_DYNAMIC;
73 }
74
75 class LibaomAv1Encoder final : public VideoEncoder {
76 public:
77 LibaomAv1Encoder();
78 ~LibaomAv1Encoder();
79
80 int InitEncode(const VideoCodec* codec_settings,
81 const Settings& settings) override;
82
83 int32_t RegisterEncodeCompleteCallback(
84 EncodedImageCallback* encoded_image_callback) override;
85
86 int32_t Release() override;
87
88 int32_t Encode(const VideoFrame& frame,
89 const std::vector<VideoFrameType>* frame_types) override;
90
91 void SetRates(const RateControlParameters& parameters) override;
92
93 EncoderInfo GetEncoderInfo() const override;
94
95 private:
96 // Determine number of encoder threads to use.
97 int NumberOfThreads(int width, int height, int number_of_cores);
98
SvcEnabled() const99 bool SvcEnabled() const { return svc_params_.has_value(); }
100 // Fills svc_params_ memeber value. Returns false on error.
101 bool SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config);
102 // Configures the encoder with layer for the next frame.
103 void SetSvcLayerId(
104 const ScalableVideoController::LayerFrameConfig& layer_frame);
105 // Configures the encoder which buffers next frame updates and can reference.
106 void SetSvcRefFrameConfig(
107 const ScalableVideoController::LayerFrameConfig& layer_frame);
108
109 std::unique_ptr<ScalableVideoController> svc_controller_;
110 bool inited_;
111 absl::optional<aom_svc_params_t> svc_params_;
112 VideoCodec encoder_settings_;
113 aom_image_t* frame_for_encode_;
114 aom_codec_ctx_t ctx_;
115 aom_codec_enc_cfg_t cfg_;
116 EncodedImageCallback* encoded_image_callback_;
117 };
118
VerifyCodecSettings(const VideoCodec & codec_settings)119 int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
120 if (codec_settings.width < 1) {
121 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
122 }
123 if (codec_settings.height < 1) {
124 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
125 }
126 // maxBitrate == 0 represents an unspecified maxBitRate.
127 if (codec_settings.maxBitrate > 0 &&
128 codec_settings.minBitrate > codec_settings.maxBitrate) {
129 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
130 }
131 if (codec_settings.maxBitrate > 0 &&
132 codec_settings.startBitrate > codec_settings.maxBitrate) {
133 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
134 }
135 if (codec_settings.startBitrate < codec_settings.minBitrate) {
136 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
137 }
138 if (codec_settings.maxFramerate < 1) {
139 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
140 }
141 return WEBRTC_VIDEO_CODEC_OK;
142 }
143
LibaomAv1Encoder()144 LibaomAv1Encoder::LibaomAv1Encoder()
145 : inited_(false),
146 frame_for_encode_(nullptr),
147 encoded_image_callback_(nullptr) {}
148
~LibaomAv1Encoder()149 LibaomAv1Encoder::~LibaomAv1Encoder() {
150 Release();
151 }
152
InitEncode(const VideoCodec * codec_settings,const Settings & settings)153 int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
154 const Settings& settings) {
155 if (codec_settings == nullptr) {
156 RTC_LOG(LS_WARNING) << "No codec settings provided to "
157 "LibaomAv1Encoder.";
158 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
159 }
160 if (settings.number_of_cores < 1) {
161 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
162 }
163 if (inited_) {
164 RTC_LOG(LS_WARNING) << "Initing LibaomAv1Encoder without first releasing.";
165 Release();
166 }
167 encoder_settings_ = *codec_settings;
168
169 // Sanity checks for encoder configuration.
170 const int32_t result = VerifyCodecSettings(encoder_settings_);
171 if (result < 0) {
172 RTC_LOG(LS_WARNING) << "Incorrect codec settings provided to "
173 "LibaomAv1Encoder.";
174 return result;
175 }
176 if (encoder_settings_.numberOfSimulcastStreams > 1) {
177 RTC_LOG(LS_WARNING) << "Simulcast is not implemented by LibaomAv1Encoder.";
178 return result;
179 }
180 absl::string_view scalability_mode = encoder_settings_.ScalabilityMode();
181 if (scalability_mode.empty()) {
182 RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'NONE'.";
183 scalability_mode = "NONE";
184 }
185 svc_controller_ = CreateScalabilityStructure(scalability_mode);
186 if (svc_controller_ == nullptr) {
187 RTC_LOG(LS_WARNING) << "Failed to set scalability mode "
188 << scalability_mode;
189 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
190 }
191
192 if (!SetSvcParams(svc_controller_->StreamConfig())) {
193 return WEBRTC_VIDEO_CODEC_ERROR;
194 }
195
196 // Initialize encoder configuration structure with default values
197 aom_codec_err_t ret =
198 aom_codec_enc_config_default(aom_codec_av1_cx(), &cfg_, kUsageProfile);
199 if (ret != AOM_CODEC_OK) {
200 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
201 << " on aom_codec_enc_config_default.";
202 return WEBRTC_VIDEO_CODEC_ERROR;
203 }
204
205 // Overwrite default config with input encoder settings & RTC-relevant values.
206 cfg_.g_w = encoder_settings_.width;
207 cfg_.g_h = encoder_settings_.height;
208 cfg_.g_threads =
209 NumberOfThreads(cfg_.g_w, cfg_.g_h, settings.number_of_cores);
210 cfg_.g_timebase.num = 1;
211 cfg_.g_timebase.den = kRtpTicksPerSecond;
212 cfg_.rc_target_bitrate = encoder_settings_.maxBitrate; // kilobits/sec.
213 cfg_.g_input_bit_depth = kBitDepth;
214 cfg_.kf_mode = AOM_KF_DISABLED;
215 cfg_.rc_min_quantizer = kQpMin;
216 cfg_.rc_max_quantizer = encoder_settings_.qpMax;
217 cfg_.g_usage = kUsageProfile;
218 cfg_.g_error_resilient = 0;
219 // Low-latency settings.
220 cfg_.rc_end_usage = AOM_CBR; // Constant Bit Rate (CBR) mode
221 cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control
222 cfg_.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0.
223
224 // Creating a wrapper to the image - setting image data to nullptr. Actual
225 // pointer will be set in encode. Setting align to 1, as it is meaningless
226 // (actual memory is not allocated).
227 frame_for_encode_ =
228 aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1);
229
230 // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
231 aom_codec_flags_t flags = 0;
232
233 // Initialize an encoder instance.
234 ret = aom_codec_enc_init(&ctx_, aom_codec_av1_cx(), &cfg_, flags);
235 if (ret != AOM_CODEC_OK) {
236 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
237 << " on aom_codec_enc_init.";
238 return WEBRTC_VIDEO_CODEC_ERROR;
239 }
240 inited_ = true;
241
242 // Set control parameters
243 ret = aom_codec_control(
244 &ctx_, AOME_SET_CPUUSED,
245 GetCpuSpeed(cfg_.g_w, cfg_.g_h, settings.number_of_cores));
246 if (ret != AOM_CODEC_OK) {
247 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
248 << " on control AV1E_SET_CPUUSED.";
249 return WEBRTC_VIDEO_CODEC_ERROR;
250 }
251 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_CDEF, 1);
252 if (ret != AOM_CODEC_OK) {
253 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
254 << " on control AV1E_SET_ENABLE_CDEF.";
255 return WEBRTC_VIDEO_CODEC_ERROR;
256 }
257 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_TPL_MODEL, 0);
258 if (ret != AOM_CODEC_OK) {
259 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
260 << " on control AV1E_SET_ENABLE_TPL_MODEL.";
261 return WEBRTC_VIDEO_CODEC_ERROR;
262 }
263 ret = aom_codec_control(&ctx_, AV1E_SET_DELTAQ_MODE, 0);
264 if (ret != AOM_CODEC_OK) {
265 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
266 << " on control AV1E_SET_DELTAQ_MODE.";
267 return WEBRTC_VIDEO_CODEC_ERROR;
268 }
269 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ORDER_HINT, 0);
270 if (ret != AOM_CODEC_OK) {
271 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
272 << " on control AV1E_SET_ENABLE_ORDER_HINT.";
273 return WEBRTC_VIDEO_CODEC_ERROR;
274 }
275 ret = aom_codec_control(&ctx_, AV1E_SET_AQ_MODE, 3);
276 if (ret != AOM_CODEC_OK) {
277 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
278 << " on control AV1E_SET_AQ_MODE.";
279 return WEBRTC_VIDEO_CODEC_ERROR;
280 }
281 if (SvcEnabled()) {
282 ret = aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
283 if (ret != AOM_CODEC_OK) {
284 RTC_LOG(LS_WARNING) << "LibaomAV1Encoder::EncodeInit returned " << ret
285 << " on control AV1E_SET_SVC_PARAMS.";
286 return false;
287 }
288 }
289
290 ret = aom_codec_control(&ctx_, AOME_SET_MAX_INTRA_BITRATE_PCT, 300);
291 if (ret != AOM_CODEC_OK) {
292 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
293 << " on control AV1E_SET_MAX_INTRA_BITRATE_PCT.";
294 return WEBRTC_VIDEO_CODEC_ERROR;
295 }
296 ret = aom_codec_control(&ctx_, AV1E_SET_COEFF_COST_UPD_FREQ, 3);
297 if (ret != AOM_CODEC_OK) {
298 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
299 << " on control AV1E_SET_COEFF_COST_UPD_FREQ.";
300 return WEBRTC_VIDEO_CODEC_ERROR;
301 }
302 ret = aom_codec_control(&ctx_, AV1E_SET_MODE_COST_UPD_FREQ, 3);
303 if (ret != AOM_CODEC_OK) {
304 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
305 << " on control AV1E_SET_MODE_COST_UPD_FREQ.";
306 return WEBRTC_VIDEO_CODEC_ERROR;
307 }
308 ret = aom_codec_control(&ctx_, AV1E_SET_MV_COST_UPD_FREQ, 3);
309 if (ret != AOM_CODEC_OK) {
310 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
311 << " on control AV1E_SET_MV_COST_UPD_FREQ.";
312 return WEBRTC_VIDEO_CODEC_ERROR;
313 }
314
315 if (cfg_.g_threads == 4 && cfg_.g_w == 640 &&
316 (cfg_.g_h == 360 || cfg_.g_h == 480)) {
317 ret = aom_codec_control(&ctx_, AV1E_SET_TILE_ROWS,
318 static_cast<int>(log2(cfg_.g_threads)));
319 if (ret != AOM_CODEC_OK) {
320 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
321 << " on control AV1E_SET_TILE_ROWS.";
322 return WEBRTC_VIDEO_CODEC_ERROR;
323 }
324 } else {
325 ret = aom_codec_control(&ctx_, AV1E_SET_TILE_COLUMNS,
326 static_cast<int>(log2(cfg_.g_threads)));
327 if (ret != AOM_CODEC_OK) {
328 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
329 << " on control AV1E_SET_TILE_COLUMNS.";
330 return WEBRTC_VIDEO_CODEC_ERROR;
331 }
332 }
333
334 ret = aom_codec_control(&ctx_, AV1E_SET_ROW_MT, 1);
335 if (ret != AOM_CODEC_OK) {
336 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
337 << " on control AV1E_SET_ROW_MT.";
338 return WEBRTC_VIDEO_CODEC_ERROR;
339 }
340
341 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_OBMC, 0);
342 if (ret != AOM_CODEC_OK) {
343 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
344 << " on control AV1E_SET_ENABLE_OBMC.";
345 return WEBRTC_VIDEO_CODEC_ERROR;
346 }
347
348 ret = aom_codec_control(&ctx_, AV1E_SET_NOISE_SENSITIVITY, 0);
349 if (ret != AOM_CODEC_OK) {
350 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
351 << " on control AV1E_SET_NOISE_SENSITIVITY.";
352 return WEBRTC_VIDEO_CODEC_ERROR;
353 }
354
355 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_WARPED_MOTION, 0);
356 if (ret != AOM_CODEC_OK) {
357 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
358 << " on control AV1E_SET_ENABLE_WARPED_MOTION.";
359 return WEBRTC_VIDEO_CODEC_ERROR;
360 }
361
362 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_GLOBAL_MOTION, 0);
363 if (ret != AOM_CODEC_OK) {
364 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
365 << " on control AV1E_SET_ENABLE_GLOBAL_MOTION.";
366 return WEBRTC_VIDEO_CODEC_ERROR;
367 }
368
369 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_REF_FRAME_MVS, 0);
370 if (ret != AOM_CODEC_OK) {
371 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
372 << " on control AV1E_SET_ENABLE_REF_FRAME_MVS.";
373 return WEBRTC_VIDEO_CODEC_ERROR;
374 }
375
376 ret =
377 aom_codec_control(&ctx_, AV1E_SET_SUPERBLOCK_SIZE,
378 GetSuperblockSize(cfg_.g_w, cfg_.g_h, cfg_.g_threads));
379 if (ret != AOM_CODEC_OK) {
380 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
381 << " on control AV1E_SET_SUPERBLOCK_SIZE.";
382 return WEBRTC_VIDEO_CODEC_ERROR;
383 }
384
385 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_CFL_INTRA, 0);
386 if (ret != AOM_CODEC_OK) {
387 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
388 << " on control AV1E_SET_ENABLE_CFL_INTRA.";
389 return WEBRTC_VIDEO_CODEC_ERROR;
390 }
391
392 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_SMOOTH_INTRA, 0);
393 if (ret != AOM_CODEC_OK) {
394 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
395 << " on control AV1E_SET_ENABLE_SMOOTH_INTRA.";
396 return WEBRTC_VIDEO_CODEC_ERROR;
397 }
398
399 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ANGLE_DELTA, 0);
400 if (ret != AOM_CODEC_OK) {
401 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
402 << " on control AV1E_SET_ENABLE_ANGLE_DELTA.";
403 return WEBRTC_VIDEO_CODEC_ERROR;
404 }
405
406 ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_FILTER_INTRA, 0);
407 if (ret != AOM_CODEC_OK) {
408 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
409 << " on control AV1E_SET_ENABLE_FILTER_INTRA.";
410 return WEBRTC_VIDEO_CODEC_ERROR;
411 }
412
413 ret = aom_codec_control(&ctx_, AV1E_SET_INTRA_DEFAULT_TX_ONLY, 1);
414 if (ret != AOM_CODEC_OK) {
415 RTC_LOG(LS_WARNING)
416 << "LibaomAv1Encoder::EncodeInit returned " << ret
417 << " on control AOM_CTRL_AV1E_SET_INTRA_DEFAULT_TX_ONLY.";
418 return WEBRTC_VIDEO_CODEC_ERROR;
419 }
420
421 return WEBRTC_VIDEO_CODEC_OK;
422 }
423
NumberOfThreads(int width,int height,int number_of_cores)424 int LibaomAv1Encoder::NumberOfThreads(int width,
425 int height,
426 int number_of_cores) {
427 // Keep the number of encoder threads equal to the possible number of
428 // column/row tiles, which is (1, 2, 4, 8). See comments below for
429 // AV1E_SET_TILE_COLUMNS/ROWS.
430 if (width * height >= 640 * 360 && number_of_cores > 4) {
431 return 4;
432 } else if (width * height >= 320 * 180 && number_of_cores > 2) {
433 return 2;
434 } else {
435 // Use 2 threads for low res on ARM.
436 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
437 defined(WEBRTC_ANDROID)
438 if (width * height >= 320 * 180 && number_of_cores > 2) {
439 return 2;
440 }
441 #endif
442 // 1 thread less than VGA.
443 return 1;
444 }
445 }
446
SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config)447 bool LibaomAv1Encoder::SetSvcParams(
448 ScalableVideoController::StreamLayersConfig svc_config) {
449 bool svc_enabled =
450 svc_config.num_spatial_layers > 1 || svc_config.num_temporal_layers > 1;
451 if (!svc_enabled) {
452 svc_params_ = absl::nullopt;
453 return true;
454 }
455 if (svc_config.num_spatial_layers < 1 || svc_config.num_spatial_layers > 4) {
456 RTC_LOG(LS_WARNING) << "Av1 supports up to 4 spatial layers. "
457 << svc_config.num_spatial_layers << " configured.";
458 return false;
459 }
460 if (svc_config.num_temporal_layers < 1 ||
461 svc_config.num_temporal_layers > 8) {
462 RTC_LOG(LS_WARNING) << "Av1 supports up to 8 temporal layers. "
463 << svc_config.num_temporal_layers << " configured.";
464 return false;
465 }
466 aom_svc_params_t& svc_params = svc_params_.emplace();
467 svc_params.number_spatial_layers = svc_config.num_spatial_layers;
468 svc_params.number_temporal_layers = svc_config.num_temporal_layers;
469
470 int num_layers =
471 svc_config.num_spatial_layers * svc_config.num_temporal_layers;
472 for (int i = 0; i < num_layers; ++i) {
473 svc_params.min_quantizers[i] = kQpMin;
474 svc_params.max_quantizers[i] = encoder_settings_.qpMax;
475 }
476
477 // Assume each temporal layer doubles framerate.
478 for (int tid = 0; tid < svc_config.num_temporal_layers; ++tid) {
479 svc_params.framerate_factor[tid] =
480 1 << (svc_config.num_temporal_layers - tid - 1);
481 }
482
483 for (int sid = 0; sid < svc_config.num_spatial_layers; ++sid) {
484 svc_params.scaling_factor_num[sid] = svc_config.scaling_factor_num[sid];
485 svc_params.scaling_factor_den[sid] = svc_config.scaling_factor_den[sid];
486 }
487
488 return true;
489 }
490
SetSvcLayerId(const ScalableVideoController::LayerFrameConfig & layer_frame)491 void LibaomAv1Encoder::SetSvcLayerId(
492 const ScalableVideoController::LayerFrameConfig& layer_frame) {
493 aom_svc_layer_id_t layer_id = {};
494 layer_id.spatial_layer_id = layer_frame.SpatialId();
495 layer_id.temporal_layer_id = layer_frame.TemporalId();
496 aom_codec_err_t ret =
497 aom_codec_control(&ctx_, AV1E_SET_SVC_LAYER_ID, &layer_id);
498 if (ret != AOM_CODEC_OK) {
499 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
500 << " on control AV1E_SET_SVC_LAYER_ID.";
501 }
502 }
503
SetSvcRefFrameConfig(const ScalableVideoController::LayerFrameConfig & layer_frame)504 void LibaomAv1Encoder::SetSvcRefFrameConfig(
505 const ScalableVideoController::LayerFrameConfig& layer_frame) {
506 // Buffer name to use for each layer_frame.buffers position. In particular
507 // when there are 2 buffers are referenced, prefer name them last and golden,
508 // because av1 bitstream format has dedicated fields for these two names.
509 // See last_frame_idx and golden_frame_idx in the av1 spec
510 // https://aomediacodec.github.io/av1-spec/av1-spec.pdf
511 static constexpr int kPreferedSlotName[] = {0, // Last
512 3, // Golden
513 1, 2, 4, 5, 6};
514 static constexpr int kAv1NumBuffers = 8;
515
516 aom_svc_ref_frame_config_t ref_frame_config = {};
517 RTC_CHECK_LE(layer_frame.Buffers().size(), ABSL_ARRAYSIZE(kPreferedSlotName));
518 for (size_t i = 0; i < layer_frame.Buffers().size(); ++i) {
519 const CodecBufferUsage& buffer = layer_frame.Buffers()[i];
520 int slot_name = kPreferedSlotName[i];
521 RTC_CHECK_GE(buffer.id, 0);
522 RTC_CHECK_LT(buffer.id, kAv1NumBuffers);
523 ref_frame_config.ref_idx[slot_name] = buffer.id;
524 if (buffer.referenced) {
525 ref_frame_config.reference[slot_name] = 1;
526 }
527 if (buffer.updated) {
528 ref_frame_config.refresh[buffer.id] = 1;
529 }
530 }
531 aom_codec_err_t ret = aom_codec_control(&ctx_, AV1E_SET_SVC_REF_FRAME_CONFIG,
532 &ref_frame_config);
533 if (ret != AOM_CODEC_OK) {
534 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
535 << " on control AV1_SET_SVC_REF_FRAME_CONFIG.";
536 }
537 }
538
RegisterEncodeCompleteCallback(EncodedImageCallback * encoded_image_callback)539 int32_t LibaomAv1Encoder::RegisterEncodeCompleteCallback(
540 EncodedImageCallback* encoded_image_callback) {
541 encoded_image_callback_ = encoded_image_callback;
542 return WEBRTC_VIDEO_CODEC_OK;
543 }
544
Release()545 int32_t LibaomAv1Encoder::Release() {
546 if (frame_for_encode_ != nullptr) {
547 aom_img_free(frame_for_encode_);
548 frame_for_encode_ = nullptr;
549 }
550 if (inited_) {
551 if (aom_codec_destroy(&ctx_)) {
552 return WEBRTC_VIDEO_CODEC_MEMORY;
553 }
554 inited_ = false;
555 }
556 return WEBRTC_VIDEO_CODEC_OK;
557 }
558
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)559 int32_t LibaomAv1Encoder::Encode(
560 const VideoFrame& frame,
561 const std::vector<VideoFrameType>* frame_types) {
562 if (!inited_ || encoded_image_callback_ == nullptr) {
563 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
564 }
565
566 bool keyframe_required =
567 frame_types != nullptr &&
568 absl::c_linear_search(*frame_types, VideoFrameType::kVideoFrameKey);
569
570 std::vector<ScalableVideoController::LayerFrameConfig> layer_frames =
571 svc_controller_->NextFrameConfig(keyframe_required);
572
573 if (layer_frames.empty()) {
574 RTC_LOG(LS_ERROR) << "SVCController returned no configuration for a frame.";
575 return WEBRTC_VIDEO_CODEC_ERROR;
576 }
577
578 // Convert input frame to I420, if needed.
579 VideoFrame prepped_input_frame = frame;
580 if (prepped_input_frame.video_frame_buffer()->type() !=
581 VideoFrameBuffer::Type::kI420 &&
582 prepped_input_frame.video_frame_buffer()->type() !=
583 VideoFrameBuffer::Type::kI420A) {
584 rtc::scoped_refptr<I420BufferInterface> converted_buffer(
585 prepped_input_frame.video_frame_buffer()->ToI420());
586 // The buffer should now be a mapped I420 or I420A format, but some buffer
587 // implementations incorrectly return the wrong buffer format, such as
588 // kNative. As a workaround to this, we perform ToI420() a second time.
589 // TODO(https://crbug.com/webrtc/12602): When Android buffers have a correct
590 // ToI420() implementaion, remove his workaround.
591 if (converted_buffer->type() != VideoFrameBuffer::Type::kI420 &&
592 converted_buffer->type() != VideoFrameBuffer::Type::kI420A) {
593 converted_buffer = converted_buffer->ToI420();
594 RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 ||
595 converted_buffer->type() == VideoFrameBuffer::Type::kI420A);
596 }
597 prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(),
598 frame.render_time_ms(), frame.rotation());
599 }
600
601 // Set frame_for_encode_ data pointers and strides.
602 auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420();
603 frame_for_encode_->planes[AOM_PLANE_Y] =
604 const_cast<unsigned char*>(i420_buffer->DataY());
605 frame_for_encode_->planes[AOM_PLANE_U] =
606 const_cast<unsigned char*>(i420_buffer->DataU());
607 frame_for_encode_->planes[AOM_PLANE_V] =
608 const_cast<unsigned char*>(i420_buffer->DataV());
609 frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY();
610 frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU();
611 frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV();
612
613 const uint32_t duration =
614 kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate);
615
616 for (size_t i = 0; i < layer_frames.size(); ++i) {
617 ScalableVideoController::LayerFrameConfig& layer_frame = layer_frames[i];
618 const bool end_of_picture = i == layer_frames.size() - 1;
619
620 aom_enc_frame_flags_t flags =
621 layer_frame.IsKeyframe() ? AOM_EFLAG_FORCE_KF : 0;
622
623 if (SvcEnabled()) {
624 SetSvcLayerId(layer_frame);
625 SetSvcRefFrameConfig(layer_frame);
626
627 aom_codec_err_t ret =
628 aom_codec_control(&ctx_, AV1E_SET_ERROR_RESILIENT_MODE,
629 layer_frame.TemporalId() > 0 ? 1 : 0);
630 if (ret != AOM_CODEC_OK) {
631 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
632 << " on control AV1E_SET_ERROR_RESILIENT_MODE.";
633 return WEBRTC_VIDEO_CODEC_ERROR;
634 }
635 }
636
637 // Encode a frame.
638 aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_,
639 frame.timestamp(), duration, flags);
640 if (ret != AOM_CODEC_OK) {
641 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
642 << " on aom_codec_encode.";
643 return WEBRTC_VIDEO_CODEC_ERROR;
644 }
645
646 // Get encoded image data.
647 EncodedImage encoded_image;
648 aom_codec_iter_t iter = nullptr;
649 int data_pkt_count = 0;
650 while (const aom_codec_cx_pkt_t* pkt =
651 aom_codec_get_cx_data(&ctx_, &iter)) {
652 if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
653 if (data_pkt_count > 0) {
654 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than "
655 "one data packet for an input video frame.";
656 Release();
657 }
658 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
659 /*data=*/static_cast<const uint8_t*>(pkt->data.frame.buf),
660 /*size=*/pkt->data.frame.sz));
661
662 if ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0) {
663 layer_frame.Keyframe();
664 }
665 encoded_image._frameType = layer_frame.IsKeyframe()
666 ? VideoFrameType::kVideoFrameKey
667 : VideoFrameType::kVideoFrameDelta;
668 encoded_image.SetTimestamp(frame.timestamp());
669 encoded_image.capture_time_ms_ = frame.render_time_ms();
670 encoded_image.rotation_ = frame.rotation();
671 encoded_image.content_type_ = VideoContentType::UNSPECIFIED;
672 // If encoded image width/height info are added to aom_codec_cx_pkt_t,
673 // use those values in lieu of the values in frame.
674 encoded_image._encodedHeight = frame.height();
675 encoded_image._encodedWidth = frame.width();
676 encoded_image.timing_.flags = VideoSendTiming::kInvalid;
677 int qp = -1;
678 ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp);
679 if (ret != AOM_CODEC_OK) {
680 RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
681 << " on control AOME_GET_LAST_QUANTIZER.";
682 return WEBRTC_VIDEO_CODEC_ERROR;
683 }
684 encoded_image.qp_ = qp;
685 encoded_image.SetColorSpace(frame.color_space());
686 ++data_pkt_count;
687 }
688 }
689
690 // Deliver encoded image data.
691 if (encoded_image.size() > 0) {
692 CodecSpecificInfo codec_specific_info;
693 codec_specific_info.codecType = kVideoCodecAV1;
694 codec_specific_info.end_of_picture = end_of_picture;
695 bool is_keyframe = layer_frame.IsKeyframe();
696 codec_specific_info.generic_frame_info =
697 svc_controller_->OnEncodeDone(std::move(layer_frame));
698 if (is_keyframe && codec_specific_info.generic_frame_info) {
699 codec_specific_info.template_structure =
700 svc_controller_->DependencyStructure();
701 auto& resolutions = codec_specific_info.template_structure->resolutions;
702 if (SvcEnabled()) {
703 resolutions.resize(svc_params_->number_spatial_layers);
704 for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
705 int n = svc_params_->scaling_factor_num[sid];
706 int d = svc_params_->scaling_factor_den[sid];
707 resolutions[sid] =
708 RenderResolution(cfg_.g_w * n / d, cfg_.g_h * n / d);
709 }
710 } else {
711 resolutions = {RenderResolution(cfg_.g_w, cfg_.g_h)};
712 }
713 }
714 encoded_image_callback_->OnEncodedImage(encoded_image,
715 &codec_specific_info);
716 }
717 }
718
719 return WEBRTC_VIDEO_CODEC_OK;
720 }
721
SetRates(const RateControlParameters & parameters)722 void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
723 if (!inited_) {
724 RTC_LOG(LS_WARNING) << "SetRates() while encoder is not initialized";
725 return;
726 }
727 if (parameters.framerate_fps < kMinimumFrameRate) {
728 RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= "
729 << kMinimumFrameRate
730 << " ): " << parameters.framerate_fps;
731 return;
732 }
733 if (parameters.bitrate.get_sum_bps() == 0) {
734 RTC_LOG(LS_WARNING) << "Attempt to set target bit rate to zero";
735 return;
736 }
737
738 svc_controller_->OnRatesUpdated(parameters.bitrate);
739 cfg_.rc_target_bitrate = parameters.bitrate.get_sum_kbps();
740
741 if (SvcEnabled()) {
742 for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
743 // libaom bitrate for spatial id S and temporal id T means bitrate
744 // of frames with spatial_id=S and temporal_id<=T
745 // while `parameters.bitrate` provdies bitrate of frames with
746 // spatial_id=S and temporal_id=T
747 int accumulated_bitrate_bps = 0;
748 for (int tid = 0; tid < svc_params_->number_temporal_layers; ++tid) {
749 int layer_index = sid * svc_params_->number_temporal_layers + tid;
750 accumulated_bitrate_bps += parameters.bitrate.GetBitrate(sid, tid);
751 // `svc_params.layer_target_bitrate` expects bitrate in kbps.
752 svc_params_->layer_target_bitrate[layer_index] =
753 accumulated_bitrate_bps / 1000;
754 }
755 }
756 aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
757 }
758
759 // Set frame rate to closest integer value.
760 encoder_settings_.maxFramerate =
761 static_cast<uint32_t>(parameters.framerate_fps + 0.5);
762
763 // Update encoder context.
764 aom_codec_err_t error_code = aom_codec_enc_config_set(&ctx_, &cfg_);
765 if (error_code != AOM_CODEC_OK) {
766 RTC_LOG(LS_WARNING) << "Error configuring encoder, error code: "
767 << error_code;
768 }
769 }
770
GetEncoderInfo() const771 VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
772 EncoderInfo info;
773 info.supports_native_handle = false;
774 info.implementation_name = "libaom";
775 info.has_trusted_rate_controller = true;
776 info.is_hardware_accelerated = false;
777 info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
778 info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
779 if (SvcEnabled()) {
780 for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
781 info.fps_allocation[sid].resize(svc_params_->number_temporal_layers);
782 for (int tid = 0; tid < svc_params_->number_temporal_layers; ++tid) {
783 info.fps_allocation[sid][tid] =
784 encoder_settings_.maxFramerate / svc_params_->framerate_factor[tid];
785 }
786 }
787 }
788 return info;
789 }
790
791 } // namespace
792
793 const bool kIsLibaomAv1EncoderSupported = true;
794
CreateLibaomAv1Encoder()795 std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() {
796 return std::make_unique<LibaomAv1Encoder>();
797 }
798
799 } // namespace webrtc
800