1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
12
13 #include <stdint.h>
14
15 #include <cstdio>
16 #include <memory>
17 #include <string>
18 #include <vector>
19
20 #include "absl/strings/match.h"
21 #include "absl/types/optional.h"
22 #include "api/fec_controller_override.h"
23 #include "api/video/i420_buffer.h"
24 #include "api/video/video_bitrate_allocation.h"
25 #include "api/video/video_frame.h"
26 #include "api/video_codecs/video_codec.h"
27 #include "api/video_codecs/video_encoder.h"
28 #include "modules/video_coding/include/video_error_codes.h"
29 #include "modules/video_coding/utility/simulcast_utility.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32 #include "system_wrappers/include/field_trial.h"
33
34 namespace webrtc {
35
36 namespace {
37
38 // If forced fallback is allowed, either:
39 //
40 // 1) The forced fallback is requested if the resolution is less than or equal
41 // to |max_pixels_|. The resolution is allowed to be scaled down to
42 // |min_pixels_|.
43 //
44 // 2) The forced fallback is requested if temporal support is preferred and the
45 // SW fallback supports temporal layers while the HW encoder does not.
46
47 struct ForcedFallbackParams {
48 public:
SupportsResolutionBasedSwitchwebrtc::__anon91df58870111::ForcedFallbackParams49 bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
50 return enable_resolution_based_switch &&
51 codec.codecType == kVideoCodecVP8 &&
52 codec.numberOfSimulcastStreams <= 1 &&
53 codec.VP8().numberOfTemporalLayers == 1 &&
54 codec.width * codec.height <= max_pixels;
55 }
56
SupportsTemporalBasedSwitchwebrtc::__anon91df58870111::ForcedFallbackParams57 bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
58 return enable_temporal_based_switch &&
59 SimulcastUtility::NumberOfTemporalLayers(codec, 0) > 1;
60 }
61
62 bool enable_temporal_based_switch = false;
63 bool enable_resolution_based_switch = false;
64 int min_pixels = 320 * 180;
65 int max_pixels = 320 * 240;
66 };
67
68 const char kVp8ForceFallbackEncoderFieldTrial[] =
69 "WebRTC-VP8-Forced-Fallback-Encoder-v2";
70
ParseFallbackParamsFromFieldTrials(const VideoEncoder & main_encoder)71 absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
72 const VideoEncoder& main_encoder) {
73 const std::string field_trial =
74 webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
75 if (!absl::StartsWith(field_trial, "Enabled")) {
76 return absl::nullopt;
77 }
78
79 int max_pixels_lower_bound =
80 main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1;
81
82 ForcedFallbackParams params;
83 params.enable_resolution_based_switch = true;
84
85 int min_bps = 0;
86 if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels,
87 ¶ms.max_pixels, &min_bps) != 3) {
88 RTC_LOG(LS_WARNING)
89 << "Invalid number of forced fallback parameters provided.";
90 return absl::nullopt;
91 } else if (params.min_pixels <= 0 ||
92 params.max_pixels < max_pixels_lower_bound ||
93 params.max_pixels < params.min_pixels || min_bps <= 0) {
94 RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
95 return absl::nullopt;
96 }
97
98 return params;
99 }
100
GetForcedFallbackParams(bool prefer_temporal_support,const VideoEncoder & main_encoder)101 absl::optional<ForcedFallbackParams> GetForcedFallbackParams(
102 bool prefer_temporal_support,
103 const VideoEncoder& main_encoder) {
104 absl::optional<ForcedFallbackParams> params =
105 ParseFallbackParamsFromFieldTrials(main_encoder);
106 if (prefer_temporal_support) {
107 if (!params.has_value()) {
108 params.emplace();
109 }
110 params->enable_temporal_based_switch = prefer_temporal_support;
111 }
112 return params;
113 }
114
115 class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
116 public:
117 VideoEncoderSoftwareFallbackWrapper(
118 std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
119 std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
120 bool prefer_temporal_support);
121 ~VideoEncoderSoftwareFallbackWrapper() override;
122
123 void SetFecControllerOverride(
124 FecControllerOverride* fec_controller_override) override;
125
126 int32_t InitEncode(const VideoCodec* codec_settings,
127 const VideoEncoder::Settings& settings) override;
128
129 int32_t RegisterEncodeCompleteCallback(
130 EncodedImageCallback* callback) override;
131
132 int32_t Release() override;
133
134 int32_t Encode(const VideoFrame& frame,
135 const std::vector<VideoFrameType>* frame_types) override;
136
137 void OnPacketLossRateUpdate(float packet_loss_rate) override;
138
139 void OnRttUpdate(int64_t rtt_ms) override;
140
141 void OnLossNotification(const LossNotification& loss_notification) override;
142
143 void SetRates(const RateControlParameters& parameters) override;
144
145 EncoderInfo GetEncoderInfo() const override;
146
147 private:
148 bool InitFallbackEncoder(bool is_forced);
149 bool TryInitForcedFallbackEncoder();
150 bool IsFallbackActive() const;
151
current_encoder()152 VideoEncoder* current_encoder() {
153 switch (encoder_state_) {
154 case EncoderState::kUninitialized:
155 RTC_LOG(LS_WARNING)
156 << "Trying to access encoder in uninitialized fallback wrapper.";
157 // Return main encoder to preserve previous behavior.
158 ABSL_FALLTHROUGH_INTENDED;
159 case EncoderState::kMainEncoderUsed:
160 return encoder_.get();
161 case EncoderState::kFallbackDueToFailure:
162 case EncoderState::kForcedFallback:
163 return fallback_encoder_.get();
164 }
165 }
166
167 // Updates encoder with last observed parameters, such as callbacks, rates,
168 // etc.
169 void PrimeEncoder(VideoEncoder* encoder) const;
170
171 // Settings used in the last InitEncode call and used if a dynamic fallback to
172 // software is required.
173 VideoCodec codec_settings_;
174 absl::optional<VideoEncoder::Settings> encoder_settings_;
175
176 // The last rate control settings, if set.
177 absl::optional<RateControlParameters> rate_control_parameters_;
178
179 // The last channel parameters set.
180 absl::optional<float> packet_loss_;
181 absl::optional<int64_t> rtt_;
182 FecControllerOverride* fec_controller_override_;
183 absl::optional<LossNotification> loss_notification_;
184
185 enum class EncoderState {
186 kUninitialized,
187 kMainEncoderUsed,
188 kFallbackDueToFailure,
189 kForcedFallback
190 };
191
192 EncoderState encoder_state_;
193 const std::unique_ptr<webrtc::VideoEncoder> encoder_;
194 const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_;
195
196 EncodedImageCallback* callback_;
197
198 const absl::optional<ForcedFallbackParams> fallback_params_;
199 int32_t EncodeWithMainEncoder(const VideoFrame& frame,
200 const std::vector<VideoFrameType>* frame_types);
201 };
202
VideoEncoderSoftwareFallbackWrapper(std::unique_ptr<webrtc::VideoEncoder> sw_encoder,std::unique_ptr<webrtc::VideoEncoder> hw_encoder,bool prefer_temporal_support)203 VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
204 std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
205 std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
206 bool prefer_temporal_support)
207 : fec_controller_override_(nullptr),
208 encoder_state_(EncoderState::kUninitialized),
209 encoder_(std::move(hw_encoder)),
210 fallback_encoder_(std::move(sw_encoder)),
211 callback_(nullptr),
212 fallback_params_(
213 GetForcedFallbackParams(prefer_temporal_support, *encoder_)) {
214 RTC_DCHECK(fallback_encoder_);
215 }
216
217 VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
218 default;
219
PrimeEncoder(VideoEncoder * encoder) const220 void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder(
221 VideoEncoder* encoder) const {
222 RTC_DCHECK(encoder);
223 // Replay callback, rates, and channel parameters.
224 if (callback_) {
225 encoder->RegisterEncodeCompleteCallback(callback_);
226 }
227 if (rate_control_parameters_) {
228 encoder->SetRates(*rate_control_parameters_);
229 }
230 if (rtt_.has_value()) {
231 encoder->OnRttUpdate(rtt_.value());
232 }
233 if (packet_loss_.has_value()) {
234 encoder->OnPacketLossRateUpdate(packet_loss_.value());
235 }
236 if (fec_controller_override_) {
237 encoder->SetFecControllerOverride(fec_controller_override_);
238 }
239 if (loss_notification_.has_value()) {
240 encoder->OnLossNotification(loss_notification_.value());
241 }
242 }
243
InitFallbackEncoder(bool is_forced)244 bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) {
245 RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding.";
246
247 RTC_DCHECK(encoder_settings_.has_value());
248 const int ret = fallback_encoder_->InitEncode(&codec_settings_,
249 encoder_settings_.value());
250
251 if (ret != WEBRTC_VIDEO_CODEC_OK) {
252 RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
253 fallback_encoder_->Release();
254 return false;
255 }
256
257 if (encoder_state_ == EncoderState::kMainEncoderUsed) {
258 // Since we're switching to the fallback encoder, Release the real encoder.
259 // It may be re-initialized via InitEncode later, and it will continue to
260 // get Set calls for rates and channel parameters in the meantime.
261 encoder_->Release();
262 }
263
264 if (is_forced) {
265 encoder_state_ = EncoderState::kForcedFallback;
266 } else {
267 encoder_state_ = EncoderState::kFallbackDueToFailure;
268 }
269
270 return true;
271 }
272
SetFecControllerOverride(FecControllerOverride * fec_controller_override)273 void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride(
274 FecControllerOverride* fec_controller_override) {
275 // It is important that only one of those would ever interact with the
276 // |fec_controller_override| at a given time. This is the responsibility
277 // of |this| to maintain.
278
279 fec_controller_override_ = fec_controller_override;
280 current_encoder()->SetFecControllerOverride(fec_controller_override);
281 }
282
InitEncode(const VideoCodec * codec_settings,const VideoEncoder::Settings & settings)283 int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
284 const VideoCodec* codec_settings,
285 const VideoEncoder::Settings& settings) {
286 // Store settings, in case we need to dynamically switch to the fallback
287 // encoder after a failed Encode call.
288 codec_settings_ = *codec_settings;
289 encoder_settings_ = settings;
290 // Clear stored rate/channel parameters.
291 rate_control_parameters_ = absl::nullopt;
292
293 RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized)
294 << "InitEncode() should never be called on an active instance!";
295
296 // Try to init forced software codec if it should be used.
297 if (TryInitForcedFallbackEncoder()) {
298 PrimeEncoder(current_encoder());
299 return WEBRTC_VIDEO_CODEC_OK;
300 }
301
302 int32_t ret = encoder_->InitEncode(codec_settings, settings);
303 if (ret == WEBRTC_VIDEO_CODEC_OK) {
304 encoder_state_ = EncoderState::kMainEncoderUsed;
305 PrimeEncoder(current_encoder());
306 return ret;
307 }
308
309 // Try to instantiate software codec.
310 if (InitFallbackEncoder(/*is_forced=*/false)) {
311 PrimeEncoder(current_encoder());
312 return WEBRTC_VIDEO_CODEC_OK;
313 }
314
315 // Software encoder failed too, use original return code.
316 encoder_state_ = EncoderState::kUninitialized;
317 return ret;
318 }
319
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)320 int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
321 EncodedImageCallback* callback) {
322 callback_ = callback;
323 return current_encoder()->RegisterEncodeCompleteCallback(callback);
324 }
325
Release()326 int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
327 if (encoder_state_ == EncoderState::kUninitialized) {
328 return WEBRTC_VIDEO_CODEC_OK;
329 }
330 int32_t ret = current_encoder()->Release();
331 encoder_state_ = EncoderState::kUninitialized;
332 return ret;
333 }
334
Encode(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)335 int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
336 const VideoFrame& frame,
337 const std::vector<VideoFrameType>* frame_types) {
338 switch (encoder_state_) {
339 case EncoderState::kUninitialized:
340 return WEBRTC_VIDEO_CODEC_ERROR;
341 case EncoderState::kMainEncoderUsed: {
342 return EncodeWithMainEncoder(frame, frame_types);
343 }
344 case EncoderState::kFallbackDueToFailure:
345 case EncoderState::kForcedFallback:
346 return fallback_encoder_->Encode(frame, frame_types);
347 }
348 }
EncodeWithMainEncoder(const VideoFrame & frame,const std::vector<VideoFrameType> * frame_types)349 int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder(
350 const VideoFrame& frame,
351 const std::vector<VideoFrameType>* frame_types) {
352 int32_t ret = encoder_->Encode(frame, frame_types);
353 // If requested, try a software fallback.
354 bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
355 if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) {
356 // Start using the fallback with this frame.
357 PrimeEncoder(current_encoder());
358 if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative &&
359 fallback_encoder_->GetEncoderInfo().supports_native_handle) {
360 return fallback_encoder_->Encode(frame, frame_types);
361 } else {
362 RTC_LOG(INFO) << "Fallback encoder does not support native handle - "
363 "converting frame to I420";
364 rtc::scoped_refptr<I420BufferInterface> src_buffer =
365 frame.video_frame_buffer()->ToI420();
366 if (!src_buffer) {
367 RTC_LOG(LS_ERROR) << "Failed to convert from to I420";
368 return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
369 }
370 rtc::scoped_refptr<I420Buffer> dst_buffer =
371 I420Buffer::Create(codec_settings_.width, codec_settings_.height);
372 dst_buffer->ScaleFrom(*src_buffer);
373 VideoFrame scaled_frame = frame;
374 scaled_frame.set_video_frame_buffer(dst_buffer);
375 scaled_frame.set_update_rect(VideoFrame::UpdateRect{
376 0, 0, scaled_frame.width(), scaled_frame.height()});
377 return fallback_encoder_->Encode(scaled_frame, frame_types);
378 }
379 }
380 // Fallback encoder failed too, return original error code.
381 return ret;
382 }
383
SetRates(const RateControlParameters & parameters)384 void VideoEncoderSoftwareFallbackWrapper::SetRates(
385 const RateControlParameters& parameters) {
386 rate_control_parameters_ = parameters;
387 return current_encoder()->SetRates(parameters);
388 }
389
OnPacketLossRateUpdate(float packet_loss_rate)390 void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
391 float packet_loss_rate) {
392 packet_loss_ = packet_loss_rate;
393 current_encoder()->OnPacketLossRateUpdate(packet_loss_rate);
394 }
395
OnRttUpdate(int64_t rtt_ms)396 void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
397 rtt_ = rtt_ms;
398 current_encoder()->OnRttUpdate(rtt_ms);
399 }
400
OnLossNotification(const LossNotification & loss_notification)401 void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
402 const LossNotification& loss_notification) {
403 loss_notification_ = loss_notification;
404 current_encoder()->OnLossNotification(loss_notification);
405 }
406
GetEncoderInfo() const407 VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
408 const {
409 EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo();
410 EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
411
412 EncoderInfo info =
413 IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
414
415 if (fallback_params_.has_value()) {
416 const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
417 ? fallback_encoder_info.scaling_settings
418 : default_encoder_info.scaling_settings;
419 info.scaling_settings =
420 settings.thresholds
421 ? VideoEncoder::ScalingSettings(settings.thresholds->low,
422 settings.thresholds->high,
423 fallback_params_->min_pixels)
424 : VideoEncoder::ScalingSettings::kOff;
425 } else {
426 info.scaling_settings = default_encoder_info.scaling_settings;
427 }
428
429 return info;
430 }
431
IsFallbackActive() const432 bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const {
433 return encoder_state_ == EncoderState::kForcedFallback ||
434 encoder_state_ == EncoderState::kFallbackDueToFailure;
435 }
436
TryInitForcedFallbackEncoder()437 bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
438 if (!fallback_params_) {
439 return false;
440 }
441
442 RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized);
443
444 if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) {
445 // Settings valid, try to instantiate software codec.
446 RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
447 << codec_settings_.width << "x" << codec_settings_.height;
448 return InitFallbackEncoder(/*is_forced=*/true);
449 }
450
451 if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) {
452 // First init main encoder to see if that supports temporal layers.
453 if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) ==
454 WEBRTC_VIDEO_CODEC_OK) {
455 encoder_state_ = EncoderState::kMainEncoderUsed;
456 }
457
458 if (encoder_state_ == EncoderState::kMainEncoderUsed &&
459 encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
460 // Primary encoder already supports temporal layers, use that instead.
461 return true;
462 }
463
464 // Try to initialize fallback and check if it supports temporal layers.
465 if (fallback_encoder_->InitEncode(&codec_settings_,
466 encoder_settings_.value()) ==
467 WEBRTC_VIDEO_CODEC_OK) {
468 if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
469 // Fallback encoder available and supports temporal layers, use it!
470 if (encoder_state_ == EncoderState::kMainEncoderUsed) {
471 // Main encoder initialized but does not support temporal layers,
472 // release it again.
473 encoder_->Release();
474 }
475 encoder_state_ = EncoderState::kForcedFallback;
476 RTC_LOG(LS_INFO)
477 << "Forced switch to SW encoder due to temporal support.";
478 return true;
479 } else {
480 // Fallback encoder intialization succeeded, but it does not support
481 // temporal layers either - release it.
482 fallback_encoder_->Release();
483 }
484 }
485
486 if (encoder_state_ == EncoderState::kMainEncoderUsed) {
487 // Main encoder already initialized - make use of it.
488 RTC_LOG(LS_INFO)
489 << "Cannot fall back for temporal support since fallback that "
490 "supports is not available. Using main encoder instead.";
491 return true;
492 }
493 }
494
495 // Neither forced fallback mode supported.
496 return false;
497 }
498
499 } // namespace
500
CreateVideoEncoderSoftwareFallbackWrapper(std::unique_ptr<VideoEncoder> sw_fallback_encoder,std::unique_ptr<VideoEncoder> hw_encoder,bool prefer_temporal_support)501 std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
502 std::unique_ptr<VideoEncoder> sw_fallback_encoder,
503 std::unique_ptr<VideoEncoder> hw_encoder,
504 bool prefer_temporal_support) {
505 return std::make_unique<VideoEncoderSoftwareFallbackWrapper>(
506 std::move(sw_fallback_encoder), std::move(hw_encoder),
507 prefer_temporal_support);
508 }
509
510 } // namespace webrtc
511