1 /*
2  *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
11 
12 #include <stddef.h>
13 #include <stdint.h>
14 
15 #include <memory>
16 #include <utility>
17 #include <vector>
18 
19 #include "absl/algorithm/container.h"
20 #include "absl/base/macros.h"
21 #include "absl/types/optional.h"
22 #include "api/scoped_refptr.h"
23 #include "api/video/encoded_image.h"
24 #include "api/video/i420_buffer.h"
25 #include "api/video/video_frame.h"
26 #include "api/video_codecs/video_codec.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "modules/video_coding/include/video_codec_interface.h"
29 #include "modules/video_coding/include/video_error_codes.h"
30 #include "modules/video_coding/svc/create_scalability_structure.h"
31 #include "modules/video_coding/svc/scalable_video_controller.h"
32 #include "modules/video_coding/svc/scalable_video_controller_no_layering.h"
33 #include "rtc_base/checks.h"
34 #include "rtc_base/logging.h"
35 #include "third_party/libaom/source/libaom/aom/aom_codec.h"
36 #include "third_party/libaom/source/libaom/aom/aom_encoder.h"
37 #include "third_party/libaom/source/libaom/aom/aomcx.h"
38 
39 namespace webrtc {
40 namespace {
41 
42 // Encoder configuration parameters
43 constexpr int kQpMin = 10;
44 constexpr int kUsageProfile = AOM_USAGE_REALTIME;
45 constexpr int kMinQindex = 145;   // Min qindex threshold for QP scaling.
46 constexpr int kMaxQindex = 205;   // Max qindex threshold for QP scaling.
47 constexpr int kBitDepth = 8;
48 constexpr int kLagInFrames = 0;  // No look ahead.
49 constexpr int kRtpTicksPerSecond = 90000;
50 constexpr float kMinimumFrameRate = 1.0;
51 
52 // Only positive speeds, range for real-time coding currently is: 6 - 8.
53 // Lower means slower/better quality, higher means fastest/lower quality.
GetCpuSpeed(int width,int height,int number_of_cores)54 int GetCpuSpeed(int width, int height, int number_of_cores) {
55   // For smaller resolutions, use lower speed setting (get some coding gain at
56   // the cost of increased encoding complexity).
57   if (number_of_cores > 4 && width * height < 320 * 180)
58     return 6;
59   else if (width * height >= 1280 * 720)
60     return 9;
61   else if (width * height >= 640 * 360)
62     return 8;
63   else
64     return 7;
65 }
66 
GetSuperblockSize(int width,int height,int threads)67 aom_superblock_size_t GetSuperblockSize(int width, int height, int threads) {
68   int resolution = width * height;
69   if (threads >= 4 && resolution >= 960 * 540 && resolution < 1920 * 1080)
70     return AOM_SUPERBLOCK_SIZE_64X64;
71   else
72     return AOM_SUPERBLOCK_SIZE_DYNAMIC;
73 }
74 
75 class LibaomAv1Encoder final : public VideoEncoder {
76  public:
77   LibaomAv1Encoder();
78   ~LibaomAv1Encoder();
79 
80   int InitEncode(const VideoCodec* codec_settings,
81                  const Settings& settings) override;
82 
83   int32_t RegisterEncodeCompleteCallback(
84       EncodedImageCallback* encoded_image_callback) override;
85 
86   int32_t Release() override;
87 
88   int32_t Encode(const VideoFrame& frame,
89                  const std::vector<VideoFrameType>* frame_types) override;
90 
91   void SetRates(const RateControlParameters& parameters) override;
92 
93   EncoderInfo GetEncoderInfo() const override;
94 
95  private:
96   // Determine number of encoder threads to use.
97   int NumberOfThreads(int width, int height, int number_of_cores);
98 
SvcEnabled() const99   bool SvcEnabled() const { return svc_params_.has_value(); }
100   // Fills svc_params_ memeber value. Returns false on error.
101   bool SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config);
102   // Configures the encoder with layer for the next frame.
103   void SetSvcLayerId(
104       const ScalableVideoController::LayerFrameConfig& layer_frame);
105   // Configures the encoder which buffers next frame updates and can reference.
106   void SetSvcRefFrameConfig(
107       const ScalableVideoController::LayerFrameConfig& layer_frame);
108 
109   std::unique_ptr<ScalableVideoController> svc_controller_;
110   bool inited_;
111   absl::optional<aom_svc_params_t> svc_params_;
112   VideoCodec encoder_settings_;
113   aom_image_t* frame_for_encode_;
114   aom_codec_ctx_t ctx_;
115   aom_codec_enc_cfg_t cfg_;
116   EncodedImageCallback* encoded_image_callback_;
117 };
118 
VerifyCodecSettings(const VideoCodec & codec_settings)119 int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
120   if (codec_settings.width < 1) {
121     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
122   }
123   if (codec_settings.height < 1) {
124     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
125   }
126   // maxBitrate == 0 represents an unspecified maxBitRate.
127   if (codec_settings.maxBitrate > 0 &&
128       codec_settings.minBitrate > codec_settings.maxBitrate) {
129     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
130   }
131   if (codec_settings.maxBitrate > 0 &&
132       codec_settings.startBitrate > codec_settings.maxBitrate) {
133     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
134   }
135   if (codec_settings.startBitrate < codec_settings.minBitrate) {
136     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
137   }
138   if (codec_settings.maxFramerate < 1) {
139     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
140   }
141   return WEBRTC_VIDEO_CODEC_OK;
142 }
143 
LibaomAv1Encoder()144 LibaomAv1Encoder::LibaomAv1Encoder()
145     : inited_(false),
146       frame_for_encode_(nullptr),
147       encoded_image_callback_(nullptr) {}
148 
~LibaomAv1Encoder()149 LibaomAv1Encoder::~LibaomAv1Encoder() {
150   Release();
151 }
152 
InitEncode(const VideoCodec * codec_settings,const Settings & settings)153 int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
154                                  const Settings& settings) {
155   if (codec_settings == nullptr) {
156     RTC_LOG(LS_WARNING) << "No codec settings provided to "
157                            "LibaomAv1Encoder.";
158     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
159   }
160   if (settings.number_of_cores < 1) {
161     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
162   }
163   if (inited_) {
164     RTC_LOG(LS_WARNING) << "Initing LibaomAv1Encoder without first releasing.";
165     Release();
166   }
167   encoder_settings_ = *codec_settings;
168 
169   // Sanity checks for encoder configuration.
170   const int32_t result = VerifyCodecSettings(encoder_settings_);
171   if (result < 0) {
172     RTC_LOG(LS_WARNING) << "Incorrect codec settings provided to "
173                            "LibaomAv1Encoder.";
174     return result;
175   }
176   if (encoder_settings_.numberOfSimulcastStreams > 1) {
177     RTC_LOG(LS_WARNING) << "Simulcast is not implemented by LibaomAv1Encoder.";
178     return result;
179   }
180   absl::string_view scalability_mode = encoder_settings_.ScalabilityMode();
181   if (scalability_mode.empty()) {
182     RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'NONE'.";
183     scalability_mode = "NONE";
184   }
185   svc_controller_ = CreateScalabilityStructure(scalability_mode);
186   if (svc_controller_ == nullptr) {
187     RTC_LOG(LS_WARNING) << "Failed to set scalability mode "
188                         << scalability_mode;
189     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
190   }
191 
192   if (!SetSvcParams(svc_controller_->StreamConfig())) {
193     return WEBRTC_VIDEO_CODEC_ERROR;
194   }
195 
196   // Initialize encoder configuration structure with default values
197   aom_codec_err_t ret =
198       aom_codec_enc_config_default(aom_codec_av1_cx(), &cfg_, kUsageProfile);
199   if (ret != AOM_CODEC_OK) {
200     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
201                         << " on aom_codec_enc_config_default.";
202     return WEBRTC_VIDEO_CODEC_ERROR;
203   }
204 
205   // Overwrite default config with input encoder settings & RTC-relevant values.
206   cfg_.g_w = encoder_settings_.width;
207   cfg_.g_h = encoder_settings_.height;
208   cfg_.g_threads =
209       NumberOfThreads(cfg_.g_w, cfg_.g_h, settings.number_of_cores);
210   cfg_.g_timebase.num = 1;
211   cfg_.g_timebase.den = kRtpTicksPerSecond;
212   cfg_.rc_target_bitrate = encoder_settings_.maxBitrate;  // kilobits/sec.
213   cfg_.g_input_bit_depth = kBitDepth;
214   cfg_.kf_mode = AOM_KF_DISABLED;
215   cfg_.rc_min_quantizer = kQpMin;
216   cfg_.rc_max_quantizer = encoder_settings_.qpMax;
217   cfg_.g_usage = kUsageProfile;
218   cfg_.g_error_resilient = 0;
219   // Low-latency settings.
220   cfg_.rc_end_usage = AOM_CBR;          // Constant Bit Rate (CBR) mode
221   cfg_.g_pass = AOM_RC_ONE_PASS;        // One-pass rate control
222   cfg_.g_lag_in_frames = kLagInFrames;  // No look ahead when lag equals 0.
223 
224   // Creating a wrapper to the image - setting image data to nullptr. Actual
225   // pointer will be set in encode. Setting align to 1, as it is meaningless
226   // (actual memory is not allocated).
227   frame_for_encode_ =
228       aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1);
229 
230   // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
231   aom_codec_flags_t flags = 0;
232 
233   // Initialize an encoder instance.
234   ret = aom_codec_enc_init(&ctx_, aom_codec_av1_cx(), &cfg_, flags);
235   if (ret != AOM_CODEC_OK) {
236     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
237                         << " on aom_codec_enc_init.";
238     return WEBRTC_VIDEO_CODEC_ERROR;
239   }
240   inited_ = true;
241 
242   // Set control parameters
243   ret = aom_codec_control(
244       &ctx_, AOME_SET_CPUUSED,
245       GetCpuSpeed(cfg_.g_w, cfg_.g_h, settings.number_of_cores));
246   if (ret != AOM_CODEC_OK) {
247     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
248                         << " on control AV1E_SET_CPUUSED.";
249     return WEBRTC_VIDEO_CODEC_ERROR;
250   }
251   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_CDEF, 1);
252   if (ret != AOM_CODEC_OK) {
253     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
254                         << " on control AV1E_SET_ENABLE_CDEF.";
255     return WEBRTC_VIDEO_CODEC_ERROR;
256   }
257   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_TPL_MODEL, 0);
258   if (ret != AOM_CODEC_OK) {
259     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
260                         << " on control AV1E_SET_ENABLE_TPL_MODEL.";
261     return WEBRTC_VIDEO_CODEC_ERROR;
262   }
263   ret = aom_codec_control(&ctx_, AV1E_SET_DELTAQ_MODE, 0);
264   if (ret != AOM_CODEC_OK) {
265     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
266                         << " on control AV1E_SET_DELTAQ_MODE.";
267     return WEBRTC_VIDEO_CODEC_ERROR;
268   }
269   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ORDER_HINT, 0);
270   if (ret != AOM_CODEC_OK) {
271     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
272                         << " on control AV1E_SET_ENABLE_ORDER_HINT.";
273     return WEBRTC_VIDEO_CODEC_ERROR;
274   }
275   ret = aom_codec_control(&ctx_, AV1E_SET_AQ_MODE, 3);
276   if (ret != AOM_CODEC_OK) {
277     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
278                         << " on control AV1E_SET_AQ_MODE.";
279     return WEBRTC_VIDEO_CODEC_ERROR;
280   }
281   if (SvcEnabled()) {
282     ret = aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
283     if (ret != AOM_CODEC_OK) {
284       RTC_LOG(LS_WARNING) << "LibaomAV1Encoder::EncodeInit returned " << ret
285                           << " on control AV1E_SET_SVC_PARAMS.";
286       return false;
287     }
288   }
289 
290   ret = aom_codec_control(&ctx_, AOME_SET_MAX_INTRA_BITRATE_PCT, 300);
291   if (ret != AOM_CODEC_OK) {
292     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
293                         << " on control AV1E_SET_MAX_INTRA_BITRATE_PCT.";
294     return WEBRTC_VIDEO_CODEC_ERROR;
295   }
296   ret = aom_codec_control(&ctx_, AV1E_SET_COEFF_COST_UPD_FREQ, 3);
297   if (ret != AOM_CODEC_OK) {
298     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
299                         << " on control AV1E_SET_COEFF_COST_UPD_FREQ.";
300     return WEBRTC_VIDEO_CODEC_ERROR;
301   }
302   ret = aom_codec_control(&ctx_, AV1E_SET_MODE_COST_UPD_FREQ, 3);
303   if (ret != AOM_CODEC_OK) {
304     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
305                         << " on control AV1E_SET_MODE_COST_UPD_FREQ.";
306     return WEBRTC_VIDEO_CODEC_ERROR;
307   }
308   ret = aom_codec_control(&ctx_, AV1E_SET_MV_COST_UPD_FREQ, 3);
309   if (ret != AOM_CODEC_OK) {
310     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
311                         << " on control AV1E_SET_MV_COST_UPD_FREQ.";
312     return WEBRTC_VIDEO_CODEC_ERROR;
313   }
314 
315   if (cfg_.g_threads == 4 && cfg_.g_w == 640 &&
316       (cfg_.g_h == 360 || cfg_.g_h == 480)) {
317     ret = aom_codec_control(&ctx_, AV1E_SET_TILE_ROWS,
318                             static_cast<int>(log2(cfg_.g_threads)));
319     if (ret != AOM_CODEC_OK) {
320       RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
321                           << " on control AV1E_SET_TILE_ROWS.";
322       return WEBRTC_VIDEO_CODEC_ERROR;
323     }
324   } else {
325     ret = aom_codec_control(&ctx_, AV1E_SET_TILE_COLUMNS,
326                             static_cast<int>(log2(cfg_.g_threads)));
327     if (ret != AOM_CODEC_OK) {
328       RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
329                           << " on control AV1E_SET_TILE_COLUMNS.";
330       return WEBRTC_VIDEO_CODEC_ERROR;
331     }
332   }
333 
334   ret = aom_codec_control(&ctx_, AV1E_SET_ROW_MT, 1);
335   if (ret != AOM_CODEC_OK) {
336     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
337                         << " on control AV1E_SET_ROW_MT.";
338     return WEBRTC_VIDEO_CODEC_ERROR;
339   }
340 
341   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_OBMC, 0);
342   if (ret != AOM_CODEC_OK) {
343     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
344                         << " on control AV1E_SET_ENABLE_OBMC.";
345     return WEBRTC_VIDEO_CODEC_ERROR;
346   }
347 
348   ret = aom_codec_control(&ctx_, AV1E_SET_NOISE_SENSITIVITY, 0);
349   if (ret != AOM_CODEC_OK) {
350     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
351                         << " on control AV1E_SET_NOISE_SENSITIVITY.";
352     return WEBRTC_VIDEO_CODEC_ERROR;
353   }
354 
355   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_WARPED_MOTION, 0);
356   if (ret != AOM_CODEC_OK) {
357     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
358                         << " on control AV1E_SET_ENABLE_WARPED_MOTION.";
359     return WEBRTC_VIDEO_CODEC_ERROR;
360   }
361 
362   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_GLOBAL_MOTION, 0);
363   if (ret != AOM_CODEC_OK) {
364     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
365                         << " on control AV1E_SET_ENABLE_GLOBAL_MOTION.";
366     return WEBRTC_VIDEO_CODEC_ERROR;
367   }
368 
369   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_REF_FRAME_MVS, 0);
370   if (ret != AOM_CODEC_OK) {
371     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
372                         << " on control AV1E_SET_ENABLE_REF_FRAME_MVS.";
373     return WEBRTC_VIDEO_CODEC_ERROR;
374   }
375 
376   ret =
377       aom_codec_control(&ctx_, AV1E_SET_SUPERBLOCK_SIZE,
378                         GetSuperblockSize(cfg_.g_w, cfg_.g_h, cfg_.g_threads));
379   if (ret != AOM_CODEC_OK) {
380     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
381                         << " on control AV1E_SET_SUPERBLOCK_SIZE.";
382     return WEBRTC_VIDEO_CODEC_ERROR;
383   }
384 
385   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_CFL_INTRA, 0);
386   if (ret != AOM_CODEC_OK) {
387     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
388                         << " on control AV1E_SET_ENABLE_CFL_INTRA.";
389     return WEBRTC_VIDEO_CODEC_ERROR;
390   }
391 
392   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_SMOOTH_INTRA, 0);
393   if (ret != AOM_CODEC_OK) {
394     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
395                         << " on control AV1E_SET_ENABLE_SMOOTH_INTRA.";
396     return WEBRTC_VIDEO_CODEC_ERROR;
397   }
398 
399   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ANGLE_DELTA, 0);
400   if (ret != AOM_CODEC_OK) {
401     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
402                         << " on control AV1E_SET_ENABLE_ANGLE_DELTA.";
403     return WEBRTC_VIDEO_CODEC_ERROR;
404   }
405 
406   ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_FILTER_INTRA, 0);
407   if (ret != AOM_CODEC_OK) {
408     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
409                         << " on control AV1E_SET_ENABLE_FILTER_INTRA.";
410     return WEBRTC_VIDEO_CODEC_ERROR;
411   }
412 
413   ret = aom_codec_control(&ctx_, AV1E_SET_INTRA_DEFAULT_TX_ONLY, 1);
414   if (ret != AOM_CODEC_OK) {
415     RTC_LOG(LS_WARNING)
416         << "LibaomAv1Encoder::EncodeInit returned " << ret
417         << " on control AOM_CTRL_AV1E_SET_INTRA_DEFAULT_TX_ONLY.";
418     return WEBRTC_VIDEO_CODEC_ERROR;
419   }
420 
421   return WEBRTC_VIDEO_CODEC_OK;
422 }
423 
NumberOfThreads(int width,int height,int number_of_cores)424 int LibaomAv1Encoder::NumberOfThreads(int width,
425                                       int height,
426                                       int number_of_cores) {
427   // Keep the number of encoder threads equal to the possible number of
428   // column/row tiles, which is (1, 2, 4, 8). See comments below for
429   // AV1E_SET_TILE_COLUMNS/ROWS.
430   if (width * height >= 640 * 360 && number_of_cores > 4) {
431     return 4;
432   } else if (width * height >= 320 * 180 && number_of_cores > 2) {
433     return 2;
434   } else {
435 // Use 2 threads for low res on ARM.
436 #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
437     defined(WEBRTC_ANDROID)
438     if (width * height >= 320 * 180 && number_of_cores > 2) {
439       return 2;
440     }
441 #endif
442     // 1 thread less than VGA.
443     return 1;
444   }
445 }
446 
SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config)447 bool LibaomAv1Encoder::SetSvcParams(
448     ScalableVideoController::StreamLayersConfig svc_config) {
449   bool svc_enabled =
450       svc_config.num_spatial_layers > 1 || svc_config.num_temporal_layers > 1;
451   if (!svc_enabled) {
452     svc_params_ = absl::nullopt;
453     return true;
454   }
455   if (svc_config.num_spatial_layers < 1 || svc_config.num_spatial_layers > 4) {
456     RTC_LOG(LS_WARNING) << "Av1 supports up to 4 spatial layers. "
457                         << svc_config.num_spatial_layers << " configured.";
458     return false;
459   }
460   if (svc_config.num_temporal_layers < 1 ||
461       svc_config.num_temporal_layers > 8) {
462     RTC_LOG(LS_WARNING) << "Av1 supports up to 8 temporal layers. "
463                         << svc_config.num_temporal_layers << " configured.";
464     return false;
465   }
466   aom_svc_params_t& svc_params = svc_params_.emplace();
467   svc_params.number_spatial_layers = svc_config.num_spatial_layers;
468   svc_params.number_temporal_layers = svc_config.num_temporal_layers;
469 
470   int num_layers =
471       svc_config.num_spatial_layers * svc_config.num_temporal_layers;
472   for (int i = 0; i < num_layers; ++i) {
473     svc_params.min_quantizers[i] = kQpMin;
474     svc_params.max_quantizers[i] = encoder_settings_.qpMax;
475   }
476 
477   // Assume each temporal layer doubles framerate.
478   for (int tid = 0; tid < svc_config.num_temporal_layers; ++tid) {
479     svc_params.framerate_factor[tid] =
480         1 << (svc_config.num_temporal_layers - tid - 1);
481   }
482 
483   for (int sid = 0; sid < svc_config.num_spatial_layers; ++sid) {
484     svc_params.scaling_factor_num[sid] = svc_config.scaling_factor_num[sid];
485     svc_params.scaling_factor_den[sid] = svc_config.scaling_factor_den[sid];
486   }
487 
488   return true;
489 }
490 
SetSvcLayerId(const ScalableVideoController::LayerFrameConfig & layer_frame)491 void LibaomAv1Encoder::SetSvcLayerId(
492     const ScalableVideoController::LayerFrameConfig& layer_frame) {
493   aom_svc_layer_id_t layer_id = {};
494   layer_id.spatial_layer_id = layer_frame.SpatialId();
495   layer_id.temporal_layer_id = layer_frame.TemporalId();
496   aom_codec_err_t ret =
497       aom_codec_control(&ctx_, AV1E_SET_SVC_LAYER_ID, &layer_id);
498   if (ret != AOM_CODEC_OK) {
499     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
500                         << " on control AV1E_SET_SVC_LAYER_ID.";
501   }
502 }
503 
SetSvcRefFrameConfig(const ScalableVideoController::LayerFrameConfig & layer_frame)504 void LibaomAv1Encoder::SetSvcRefFrameConfig(
505     const ScalableVideoController::LayerFrameConfig& layer_frame) {
506   // Buffer name to use for each layer_frame.buffers position. In particular
507   // when there are 2 buffers are referenced, prefer name them last and golden,
508   // because av1 bitstream format has dedicated fields for these two names.
509   // See last_frame_idx and golden_frame_idx in the av1 spec
510   // https://aomediacodec.github.io/av1-spec/av1-spec.pdf
511   static constexpr int kPreferedSlotName[] = {0,  // Last
512                                               3,  // Golden
513                                               1, 2, 4, 5, 6};
514   static constexpr int kAv1NumBuffers = 8;
515 
516   aom_svc_ref_frame_config_t ref_frame_config = {};
517   RTC_CHECK_LE(layer_frame.Buffers().size(), ABSL_ARRAYSIZE(kPreferedSlotName));
518   for (size_t i = 0; i < layer_frame.Buffers().size(); ++i) {
519     const CodecBufferUsage& buffer = layer_frame.Buffers()[i];
520     int slot_name = kPreferedSlotName[i];
521     RTC_CHECK_GE(buffer.id, 0);
522     RTC_CHECK_LT(buffer.id, kAv1NumBuffers);
523     ref_frame_config.ref_idx[slot_name] = buffer.id;
524     if (buffer.referenced) {
525       ref_frame_config.reference[slot_name] = 1;
526     }
527     if (buffer.updated) {
528       ref_frame_config.refresh[buffer.id] = 1;
529     }
530   }
531   aom_codec_err_t ret = aom_codec_control(&ctx_, AV1E_SET_SVC_REF_FRAME_CONFIG,
532                                           &ref_frame_config);
533   if (ret != AOM_CODEC_OK) {
534     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
535                         << " on control AV1_SET_SVC_REF_FRAME_CONFIG.";
536   }
537 }
538 
RegisterEncodeCompleteCallback(EncodedImageCallback * encoded_image_callback)539 int32_t LibaomAv1Encoder::RegisterEncodeCompleteCallback(
540     EncodedImageCallback* encoded_image_callback) {
541   encoded_image_callback_ = encoded_image_callback;
542   return WEBRTC_VIDEO_CODEC_OK;
543 }
544 
Release()545 int32_t LibaomAv1Encoder::Release() {
546   if (frame_for_encode_ != nullptr) {
547     aom_img_free(frame_for_encode_);
548     frame_for_encode_ = nullptr;
549   }
550   if (inited_) {
551     if (aom_codec_destroy(&ctx_)) {
552       return WEBRTC_VIDEO_CODEC_MEMORY;
553     }
554     inited_ = false;
555   }
556   return WEBRTC_VIDEO_CODEC_OK;
557 }
558 
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)559 int32_t LibaomAv1Encoder::Encode(
560     const VideoFrame& frame,
561     const std::vector<VideoFrameType>* frame_types) {
562   if (!inited_ || encoded_image_callback_ == nullptr) {
563     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
564   }
565 
566   bool keyframe_required =
567       frame_types != nullptr &&
568       absl::c_linear_search(*frame_types, VideoFrameType::kVideoFrameKey);
569 
570   std::vector<ScalableVideoController::LayerFrameConfig> layer_frames =
571       svc_controller_->NextFrameConfig(keyframe_required);
572 
573   if (layer_frames.empty()) {
574     RTC_LOG(LS_ERROR) << "SVCController returned no configuration for a frame.";
575     return WEBRTC_VIDEO_CODEC_ERROR;
576   }
577 
578   // Convert input frame to I420, if needed.
579   VideoFrame prepped_input_frame = frame;
580   if (prepped_input_frame.video_frame_buffer()->type() !=
581           VideoFrameBuffer::Type::kI420 &&
582       prepped_input_frame.video_frame_buffer()->type() !=
583           VideoFrameBuffer::Type::kI420A) {
584     rtc::scoped_refptr<I420BufferInterface> converted_buffer(
585         prepped_input_frame.video_frame_buffer()->ToI420());
586     // The buffer should now be a mapped I420 or I420A format, but some buffer
587     // implementations incorrectly return the wrong buffer format, such as
588     // kNative. As a workaround to this, we perform ToI420() a second time.
589     // TODO(https://crbug.com/webrtc/12602): When Android buffers have a correct
590     // ToI420() implementaion, remove his workaround.
591     if (converted_buffer->type() != VideoFrameBuffer::Type::kI420 &&
592         converted_buffer->type() != VideoFrameBuffer::Type::kI420A) {
593       converted_buffer = converted_buffer->ToI420();
594       RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 ||
595                 converted_buffer->type() == VideoFrameBuffer::Type::kI420A);
596     }
597     prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(),
598                                      frame.render_time_ms(), frame.rotation());
599   }
600 
601   // Set frame_for_encode_ data pointers and strides.
602   auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420();
603   frame_for_encode_->planes[AOM_PLANE_Y] =
604       const_cast<unsigned char*>(i420_buffer->DataY());
605   frame_for_encode_->planes[AOM_PLANE_U] =
606       const_cast<unsigned char*>(i420_buffer->DataU());
607   frame_for_encode_->planes[AOM_PLANE_V] =
608       const_cast<unsigned char*>(i420_buffer->DataV());
609   frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY();
610   frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU();
611   frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV();
612 
613   const uint32_t duration =
614       kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate);
615 
616   for (size_t i = 0; i < layer_frames.size(); ++i) {
617     ScalableVideoController::LayerFrameConfig& layer_frame = layer_frames[i];
618     const bool end_of_picture = i == layer_frames.size() - 1;
619 
620     aom_enc_frame_flags_t flags =
621         layer_frame.IsKeyframe() ? AOM_EFLAG_FORCE_KF : 0;
622 
623     if (SvcEnabled()) {
624       SetSvcLayerId(layer_frame);
625       SetSvcRefFrameConfig(layer_frame);
626 
627       aom_codec_err_t ret =
628           aom_codec_control(&ctx_, AV1E_SET_ERROR_RESILIENT_MODE,
629                             layer_frame.TemporalId() > 0 ? 1 : 0);
630       if (ret != AOM_CODEC_OK) {
631         RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
632                             << " on control AV1E_SET_ERROR_RESILIENT_MODE.";
633         return WEBRTC_VIDEO_CODEC_ERROR;
634       }
635     }
636 
637     // Encode a frame.
638     aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_,
639                                            frame.timestamp(), duration, flags);
640     if (ret != AOM_CODEC_OK) {
641       RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
642                           << " on aom_codec_encode.";
643       return WEBRTC_VIDEO_CODEC_ERROR;
644     }
645 
646     // Get encoded image data.
647     EncodedImage encoded_image;
648     aom_codec_iter_t iter = nullptr;
649     int data_pkt_count = 0;
650     while (const aom_codec_cx_pkt_t* pkt =
651                aom_codec_get_cx_data(&ctx_, &iter)) {
652       if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
653         if (data_pkt_count > 0) {
654           RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than "
655                                  "one data packet for an input video frame.";
656           Release();
657         }
658         encoded_image.SetEncodedData(EncodedImageBuffer::Create(
659             /*data=*/static_cast<const uint8_t*>(pkt->data.frame.buf),
660             /*size=*/pkt->data.frame.sz));
661 
662         if ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0) {
663           layer_frame.Keyframe();
664         }
665         encoded_image._frameType = layer_frame.IsKeyframe()
666                                        ? VideoFrameType::kVideoFrameKey
667                                        : VideoFrameType::kVideoFrameDelta;
668         encoded_image.SetTimestamp(frame.timestamp());
669         encoded_image.capture_time_ms_ = frame.render_time_ms();
670         encoded_image.rotation_ = frame.rotation();
671         encoded_image.content_type_ = VideoContentType::UNSPECIFIED;
672         // If encoded image width/height info are added to aom_codec_cx_pkt_t,
673         // use those values in lieu of the values in frame.
674         encoded_image._encodedHeight = frame.height();
675         encoded_image._encodedWidth = frame.width();
676         encoded_image.timing_.flags = VideoSendTiming::kInvalid;
677         int qp = -1;
678         ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp);
679         if (ret != AOM_CODEC_OK) {
680           RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
681                               << " on control AOME_GET_LAST_QUANTIZER.";
682           return WEBRTC_VIDEO_CODEC_ERROR;
683         }
684         encoded_image.qp_ = qp;
685         encoded_image.SetColorSpace(frame.color_space());
686         ++data_pkt_count;
687       }
688     }
689 
690     // Deliver encoded image data.
691     if (encoded_image.size() > 0) {
692       CodecSpecificInfo codec_specific_info;
693       codec_specific_info.codecType = kVideoCodecAV1;
694       codec_specific_info.end_of_picture = end_of_picture;
695       bool is_keyframe = layer_frame.IsKeyframe();
696       codec_specific_info.generic_frame_info =
697           svc_controller_->OnEncodeDone(std::move(layer_frame));
698       if (is_keyframe && codec_specific_info.generic_frame_info) {
699         codec_specific_info.template_structure =
700             svc_controller_->DependencyStructure();
701         auto& resolutions = codec_specific_info.template_structure->resolutions;
702         if (SvcEnabled()) {
703           resolutions.resize(svc_params_->number_spatial_layers);
704           for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
705             int n = svc_params_->scaling_factor_num[sid];
706             int d = svc_params_->scaling_factor_den[sid];
707             resolutions[sid] =
708                 RenderResolution(cfg_.g_w * n / d, cfg_.g_h * n / d);
709           }
710         } else {
711           resolutions = {RenderResolution(cfg_.g_w, cfg_.g_h)};
712         }
713       }
714       encoded_image_callback_->OnEncodedImage(encoded_image,
715                                               &codec_specific_info);
716     }
717   }
718 
719   return WEBRTC_VIDEO_CODEC_OK;
720 }
721 
SetRates(const RateControlParameters & parameters)722 void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
723   if (!inited_) {
724     RTC_LOG(LS_WARNING) << "SetRates() while encoder is not initialized";
725     return;
726   }
727   if (parameters.framerate_fps < kMinimumFrameRate) {
728     RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= "
729                         << kMinimumFrameRate
730                         << " ): " << parameters.framerate_fps;
731     return;
732   }
733   if (parameters.bitrate.get_sum_bps() == 0) {
734     RTC_LOG(LS_WARNING) << "Attempt to set target bit rate to zero";
735     return;
736   }
737 
738   svc_controller_->OnRatesUpdated(parameters.bitrate);
739   cfg_.rc_target_bitrate = parameters.bitrate.get_sum_kbps();
740 
741   if (SvcEnabled()) {
742     for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
743       // libaom bitrate for spatial id S and temporal id T means bitrate
744       // of frames with spatial_id=S and temporal_id<=T
745       // while `parameters.bitrate` provdies bitrate of frames with
746       // spatial_id=S and temporal_id=T
747       int accumulated_bitrate_bps = 0;
748       for (int tid = 0; tid < svc_params_->number_temporal_layers; ++tid) {
749         int layer_index = sid * svc_params_->number_temporal_layers + tid;
750         accumulated_bitrate_bps += parameters.bitrate.GetBitrate(sid, tid);
751         // `svc_params.layer_target_bitrate` expects bitrate in kbps.
752         svc_params_->layer_target_bitrate[layer_index] =
753             accumulated_bitrate_bps / 1000;
754       }
755     }
756     aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
757   }
758 
759   // Set frame rate to closest integer value.
760   encoder_settings_.maxFramerate =
761       static_cast<uint32_t>(parameters.framerate_fps + 0.5);
762 
763   // Update encoder context.
764   aom_codec_err_t error_code = aom_codec_enc_config_set(&ctx_, &cfg_);
765   if (error_code != AOM_CODEC_OK) {
766     RTC_LOG(LS_WARNING) << "Error configuring encoder, error code: "
767                         << error_code;
768   }
769 }
770 
GetEncoderInfo() const771 VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
772   EncoderInfo info;
773   info.supports_native_handle = false;
774   info.implementation_name = "libaom";
775   info.has_trusted_rate_controller = true;
776   info.is_hardware_accelerated = false;
777   info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
778   info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
779   if (SvcEnabled()) {
780     for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
781       info.fps_allocation[sid].resize(svc_params_->number_temporal_layers);
782       for (int tid = 0; tid < svc_params_->number_temporal_layers; ++tid) {
783         info.fps_allocation[sid][tid] =
784             encoder_settings_.maxFramerate / svc_params_->framerate_factor[tid];
785       }
786     }
787   }
788   return info;
789 }
790 
791 }  // namespace
792 
793 const bool kIsLibaomAv1EncoderSupported = true;
794 
CreateLibaomAv1Encoder()795 std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() {
796   return std::make_unique<LibaomAv1Encoder>();
797 }
798 
799 }  // namespace webrtc
800