1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "media/engine/simulcast_encoder_adapter.h"
12
13 #include <stdio.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <cstdint>
18 #include <string>
19 #include <utility>
20
21 #include "api/scoped_refptr.h"
22 #include "api/video/i420_buffer.h"
23 #include "api/video/video_codec_constants.h"
24 #include "api/video/video_frame_buffer.h"
25 #include "api/video/video_rotation.h"
26 #include "api/video_codecs/video_encoder.h"
27 #include "api/video_codecs/video_encoder_factory.h"
28 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
29 #include "media/base/video_common.h"
30 #include "modules/video_coding/include/video_error_codes.h"
31 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
32 #include "rtc_base/atomic_ops.h"
33 #include "rtc_base/checks.h"
34 #include "rtc_base/experiments/rate_control_settings.h"
35 #include "rtc_base/logging.h"
36 #include "system_wrappers/include/field_trial.h"
37
38 namespace {
39
40 const unsigned int kDefaultMinQp = 2;
41 const unsigned int kDefaultMaxQp = 56;
42 // Max qp for lowest spatial resolution when doing simulcast.
43 const unsigned int kLowestResMaxQp = 45;
44
GetScreenshareBoostedQpValue()45 absl::optional<unsigned int> GetScreenshareBoostedQpValue() {
46 std::string experiment_group =
47 webrtc::field_trial::FindFullName("WebRTC-BoostedScreenshareQp");
48 unsigned int qp;
49 if (sscanf(experiment_group.c_str(), "%u", &qp) != 1)
50 return absl::nullopt;
51 qp = std::min(qp, 63u);
52 qp = std::max(qp, 1u);
53 return qp;
54 }
55
SumStreamMaxBitrate(int streams,const webrtc::VideoCodec & codec)56 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
57 uint32_t bitrate_sum = 0;
58 for (int i = 0; i < streams; ++i) {
59 bitrate_sum += codec.simulcastStream[i].maxBitrate;
60 }
61 return bitrate_sum;
62 }
63
NumberOfStreams(const webrtc::VideoCodec & codec)64 int NumberOfStreams(const webrtc::VideoCodec& codec) {
65 int streams =
66 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
67 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
68 if (simulcast_max_bitrate == 0) {
69 streams = 1;
70 }
71 return streams;
72 }
73
NumActiveStreams(const webrtc::VideoCodec & codec)74 int NumActiveStreams(const webrtc::VideoCodec& codec) {
75 int num_configured_streams = NumberOfStreams(codec);
76 int num_active_streams = 0;
77 for (int i = 0; i < num_configured_streams; ++i) {
78 if (codec.simulcastStream[i].active) {
79 ++num_active_streams;
80 }
81 }
82 return num_active_streams;
83 }
84
VerifyCodec(const webrtc::VideoCodec * inst)85 int VerifyCodec(const webrtc::VideoCodec* inst) {
86 if (inst == nullptr) {
87 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
88 }
89 if (inst->maxFramerate < 1) {
90 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
91 }
92 // allow zero to represent an unspecified maxBitRate
93 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
94 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
95 }
96 if (inst->width <= 1 || inst->height <= 1) {
97 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
98 }
99 if (inst->codecType == webrtc::kVideoCodecVP8 &&
100 inst->VP8().automaticResizeOn && NumActiveStreams(*inst) > 1) {
101 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
102 }
103 return WEBRTC_VIDEO_CODEC_OK;
104 }
105
StreamResolutionCompare(const webrtc::SpatialLayer & a,const webrtc::SpatialLayer & b)106 bool StreamResolutionCompare(const webrtc::SpatialLayer& a,
107 const webrtc::SpatialLayer& b) {
108 return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
109 std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
110 }
111
112 // An EncodedImageCallback implementation that forwards on calls to a
113 // SimulcastEncoderAdapter, but with the stream index it's registered with as
114 // the first parameter to Encoded.
115 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback {
116 public:
AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter * adapter,size_t stream_idx)117 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter,
118 size_t stream_idx)
119 : adapter_(adapter), stream_idx_(stream_idx) {}
120
OnEncodedImage(const webrtc::EncodedImage & encoded_image,const webrtc::CodecSpecificInfo * codec_specific_info)121 EncodedImageCallback::Result OnEncodedImage(
122 const webrtc::EncodedImage& encoded_image,
123 const webrtc::CodecSpecificInfo* codec_specific_info) override {
124 return adapter_->OnEncodedImage(stream_idx_, encoded_image,
125 codec_specific_info);
126 }
127
128 private:
129 webrtc::SimulcastEncoderAdapter* const adapter_;
130 const size_t stream_idx_;
131 };
132 } // namespace
133
134 namespace webrtc {
135
SimulcastEncoderAdapter(VideoEncoderFactory * factory,const SdpVideoFormat & format)136 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory,
137 const SdpVideoFormat& format)
138 : SimulcastEncoderAdapter(factory, nullptr, format) {}
139
SimulcastEncoderAdapter(VideoEncoderFactory * primary_factory,VideoEncoderFactory * fallback_factory,const SdpVideoFormat & format)140 SimulcastEncoderAdapter::SimulcastEncoderAdapter(
141 VideoEncoderFactory* primary_factory,
142 VideoEncoderFactory* fallback_factory,
143 const SdpVideoFormat& format)
144 : inited_(0),
145 primary_encoder_factory_(primary_factory),
146 fallback_encoder_factory_(fallback_factory),
147 video_format_(format),
148 encoded_complete_callback_(nullptr),
149 experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
150 boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
151 .Vp8BoostBaseLayerQuality()),
152 prefer_temporal_support_on_base_layer_(field_trial::IsEnabled(
153 "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) {
154 RTC_DCHECK(primary_factory);
155
156 // The adapter is typically created on the worker thread, but operated on
157 // the encoder task queue.
158 encoder_queue_.Detach();
159
160 memset(&codec_, 0, sizeof(webrtc::VideoCodec));
161 }
162
~SimulcastEncoderAdapter()163 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
164 RTC_DCHECK(!Initialized());
165 DestroyStoredEncoders();
166 }
167
SetFecControllerOverride(FecControllerOverride * fec_controller_override)168 void SimulcastEncoderAdapter::SetFecControllerOverride(
169 FecControllerOverride* fec_controller_override) {
170 // Ignored.
171 }
172
Release()173 int SimulcastEncoderAdapter::Release() {
174 RTC_DCHECK_RUN_ON(&encoder_queue_);
175
176 while (!streaminfos_.empty()) {
177 std::unique_ptr<VideoEncoder> encoder =
178 std::move(streaminfos_.back().encoder);
179 // Even though it seems very unlikely, there are no guarantees that the
180 // encoder will not call back after being Release()'d. Therefore, we first
181 // disable the callbacks here.
182 encoder->RegisterEncodeCompleteCallback(nullptr);
183 encoder->Release();
184 streaminfos_.pop_back(); // Deletes callback adapter.
185 stored_encoders_.push(std::move(encoder));
186 }
187
188 // It's legal to move the encoder to another queue now.
189 encoder_queue_.Detach();
190
191 rtc::AtomicOps::ReleaseStore(&inited_, 0);
192
193 return WEBRTC_VIDEO_CODEC_OK;
194 }
195
196 // TODO(eladalon): s/inst/codec_settings/g.
InitEncode(const VideoCodec * inst,const VideoEncoder::Settings & settings)197 int SimulcastEncoderAdapter::InitEncode(
198 const VideoCodec* inst,
199 const VideoEncoder::Settings& settings) {
200 RTC_DCHECK_RUN_ON(&encoder_queue_);
201
202 if (settings.number_of_cores < 1) {
203 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
204 }
205
206 int ret = VerifyCodec(inst);
207 if (ret < 0) {
208 return ret;
209 }
210
211 ret = Release();
212 if (ret < 0) {
213 return ret;
214 }
215
216 int number_of_streams = NumberOfStreams(*inst);
217 RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
218 bool doing_simulcast_using_adapter = (number_of_streams > 1);
219 int num_active_streams = NumActiveStreams(*inst);
220
221 codec_ = *inst;
222 SimulcastRateAllocator rate_allocator(codec_);
223 VideoBitrateAllocation allocation =
224 rate_allocator.Allocate(VideoBitrateAllocationParameters(
225 codec_.startBitrate * 1000, codec_.maxFramerate));
226 std::vector<uint32_t> start_bitrates;
227 for (int i = 0; i < kMaxSimulcastStreams; ++i) {
228 uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
229 start_bitrates.push_back(stream_bitrate);
230 }
231
232 // Create |number_of_streams| of encoder instances and init them.
233 const auto minmax = std::minmax_element(
234 std::begin(codec_.simulcastStream),
235 std::begin(codec_.simulcastStream) + number_of_streams,
236 StreamResolutionCompare);
237 const auto lowest_resolution_stream_index =
238 std::distance(std::begin(codec_.simulcastStream), minmax.first);
239 const auto highest_resolution_stream_index =
240 std::distance(std::begin(codec_.simulcastStream), minmax.second);
241
242 RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
243 RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
244
245 const SdpVideoFormat format(
246 codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264",
247 video_format_.parameters);
248
249 for (int i = 0; i < number_of_streams; ++i) {
250 // If an existing encoder instance exists, reuse it.
251 // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
252 // when we start storing that state outside the encoder wrappers.
253 std::unique_ptr<VideoEncoder> encoder;
254 if (!stored_encoders_.empty()) {
255 encoder = std::move(stored_encoders_.top());
256 stored_encoders_.pop();
257 } else {
258 encoder = primary_encoder_factory_->CreateVideoEncoder(format);
259 if (fallback_encoder_factory_ != nullptr) {
260 encoder = CreateVideoEncoderSoftwareFallbackWrapper(
261 fallback_encoder_factory_->CreateVideoEncoder(format),
262 std::move(encoder),
263 i == lowest_resolution_stream_index &&
264 prefer_temporal_support_on_base_layer_);
265 }
266 }
267
268 bool encoder_initialized = false;
269 if (doing_simulcast_using_adapter && i == 0 &&
270 encoder->GetEncoderInfo().supports_simulcast) {
271 ret = encoder->InitEncode(&codec_, settings);
272 if (ret < 0) {
273 encoder->Release();
274 } else {
275 doing_simulcast_using_adapter = false;
276 number_of_streams = 1;
277 encoder_initialized = true;
278 }
279 }
280
281 VideoCodec stream_codec;
282 uint32_t start_bitrate_kbps = start_bitrates[i];
283 const bool send_stream = doing_simulcast_using_adapter
284 ? start_bitrate_kbps > 0
285 : num_active_streams > 0;
286 if (!doing_simulcast_using_adapter) {
287 stream_codec = codec_;
288 stream_codec.numberOfSimulcastStreams =
289 std::max<uint8_t>(1, stream_codec.numberOfSimulcastStreams);
290 } else {
291 // Cap start bitrate to the min bitrate in order to avoid strange codec
292 // behavior. Since sending will be false, this should not matter.
293 StreamResolution stream_resolution =
294 i == highest_resolution_stream_index
295 ? StreamResolution::HIGHEST
296 : i == lowest_resolution_stream_index ? StreamResolution::LOWEST
297 : StreamResolution::OTHER;
298
299 start_bitrate_kbps =
300 std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
301 PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
302 &stream_codec);
303 }
304
305 // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
306 if (stream_codec.qpMax < kDefaultMinQp) {
307 stream_codec.qpMax = kDefaultMaxQp;
308 }
309
310 if (!encoder_initialized) {
311 ret = encoder->InitEncode(&stream_codec, settings);
312 if (ret < 0) {
313 // Explicitly destroy the current encoder; because we haven't registered
314 // a StreamInfo for it yet, Release won't do anything about it.
315 encoder.reset();
316 Release();
317 return ret;
318 }
319 }
320
321 if (!doing_simulcast_using_adapter) {
322 // Without simulcast, just pass through the encoder info from the one
323 // active encoder.
324 encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_);
325 streaminfos_.emplace_back(
326 std::move(encoder), nullptr,
327 std::make_unique<FramerateController>(stream_codec.maxFramerate),
328 stream_codec.width, stream_codec.height, send_stream);
329 } else {
330 std::unique_ptr<EncodedImageCallback> callback(
331 new AdapterEncodedImageCallback(this, i));
332 encoder->RegisterEncodeCompleteCallback(callback.get());
333 streaminfos_.emplace_back(
334 std::move(encoder), std::move(callback),
335 std::make_unique<FramerateController>(stream_codec.maxFramerate),
336 stream_codec.width, stream_codec.height, send_stream);
337 }
338 }
339
340 // To save memory, don't store encoders that we don't use.
341 DestroyStoredEncoders();
342
343 rtc::AtomicOps::ReleaseStore(&inited_, 1);
344
345 return WEBRTC_VIDEO_CODEC_OK;
346 }
347
Encode(const VideoFrame & input_image,const std::vector<VideoFrameType> * frame_types)348 int SimulcastEncoderAdapter::Encode(
349 const VideoFrame& input_image,
350 const std::vector<VideoFrameType>* frame_types) {
351 RTC_DCHECK_RUN_ON(&encoder_queue_);
352
353 if (!Initialized()) {
354 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
355 }
356 if (encoded_complete_callback_ == nullptr) {
357 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
358 }
359
360 // All active streams should generate a key frame if
361 // a key frame is requested by any stream.
362 bool send_key_frame = false;
363 if (frame_types) {
364 for (size_t i = 0; i < frame_types->size(); ++i) {
365 if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) {
366 send_key_frame = true;
367 break;
368 }
369 }
370 }
371 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
372 if (streaminfos_[stream_idx].key_frame_request &&
373 streaminfos_[stream_idx].send_stream) {
374 send_key_frame = true;
375 break;
376 }
377 }
378
379 // Temporary thay may hold the result of texture to i420 buffer conversion.
380 rtc::scoped_refptr<I420BufferInterface> src_buffer;
381 int src_width = input_image.width();
382 int src_height = input_image.height();
383 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
384 // Don't encode frames in resolutions that we don't intend to send.
385 if (!streaminfos_[stream_idx].send_stream) {
386 continue;
387 }
388
389 const uint32_t frame_timestamp_ms =
390 1000 * input_image.timestamp() / 90000; // kVideoPayloadTypeFrequency;
391
392 // If adapter is passed through and only one sw encoder does simulcast,
393 // frame types for all streams should be passed to the encoder unchanged.
394 // Otherwise a single per-encoder frame type is passed.
395 std::vector<VideoFrameType> stream_frame_types(
396 streaminfos_.size() == 1 ? NumberOfStreams(codec_) : 1);
397 if (send_key_frame) {
398 std::fill(stream_frame_types.begin(), stream_frame_types.end(),
399 VideoFrameType::kVideoFrameKey);
400 streaminfos_[stream_idx].key_frame_request = false;
401 } else {
402 if (streaminfos_[stream_idx].framerate_controller->DropFrame(
403 frame_timestamp_ms)) {
404 continue;
405 }
406 std::fill(stream_frame_types.begin(), stream_frame_types.end(),
407 VideoFrameType::kVideoFrameDelta);
408 }
409 streaminfos_[stream_idx].framerate_controller->AddFrame(frame_timestamp_ms);
410
411 int dst_width = streaminfos_[stream_idx].width;
412 int dst_height = streaminfos_[stream_idx].height;
413 // If scaling isn't required, because the input resolution
414 // matches the destination or the input image is empty (e.g.
415 // a keyframe request for encoders with internal camera
416 // sources) or the source image has a native handle, pass the image on
417 // directly. Otherwise, we'll scale it to match what the encoder expects
418 // (below).
419 // For texture frames, the underlying encoder is expected to be able to
420 // correctly sample/scale the source texture.
421 // TODO(perkj): ensure that works going forward, and figure out how this
422 // affects webrtc:5683.
423 if ((dst_width == src_width && dst_height == src_height) ||
424 (input_image.video_frame_buffer()->type() ==
425 VideoFrameBuffer::Type::kNative &&
426 streaminfos_[stream_idx]
427 .encoder->GetEncoderInfo()
428 .supports_native_handle)) {
429 int ret = streaminfos_[stream_idx].encoder->Encode(input_image,
430 &stream_frame_types);
431 if (ret != WEBRTC_VIDEO_CODEC_OK) {
432 return ret;
433 }
434 } else {
435 if (src_buffer == nullptr) {
436 src_buffer = input_image.video_frame_buffer()->ToI420();
437 }
438 rtc::scoped_refptr<I420Buffer> dst_buffer =
439 I420Buffer::Create(dst_width, dst_height);
440
441 dst_buffer->ScaleFrom(*src_buffer);
442
443 // UpdateRect is not propagated to lower simulcast layers currently.
444 // TODO(ilnik): Consider scaling UpdateRect together with the buffer.
445 VideoFrame frame(input_image);
446 frame.set_video_frame_buffer(dst_buffer);
447 frame.set_rotation(webrtc::kVideoRotation_0);
448 frame.set_update_rect(
449 VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()});
450 int ret =
451 streaminfos_[stream_idx].encoder->Encode(frame, &stream_frame_types);
452 if (ret != WEBRTC_VIDEO_CODEC_OK) {
453 return ret;
454 }
455 }
456 }
457
458 return WEBRTC_VIDEO_CODEC_OK;
459 }
460
RegisterEncodeCompleteCallback(EncodedImageCallback * callback)461 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
462 EncodedImageCallback* callback) {
463 RTC_DCHECK_RUN_ON(&encoder_queue_);
464 encoded_complete_callback_ = callback;
465 if (streaminfos_.size() == 1) {
466 streaminfos_[0].encoder->RegisterEncodeCompleteCallback(callback);
467 }
468 return WEBRTC_VIDEO_CODEC_OK;
469 }
470
SetRates(const RateControlParameters & parameters)471 void SimulcastEncoderAdapter::SetRates(
472 const RateControlParameters& parameters) {
473 RTC_DCHECK_RUN_ON(&encoder_queue_);
474
475 if (!Initialized()) {
476 RTC_LOG(LS_WARNING) << "SetRates while not initialized";
477 return;
478 }
479
480 if (parameters.framerate_fps < 1.0) {
481 RTC_LOG(LS_WARNING) << "Invalid framerate: " << parameters.framerate_fps;
482 return;
483 }
484
485 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
486
487 if (streaminfos_.size() == 1) {
488 // Not doing simulcast.
489 streaminfos_[0].encoder->SetRates(parameters);
490 return;
491 }
492
493 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
494 uint32_t stream_bitrate_kbps =
495 parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
496
497 // Need a key frame if we have not sent this stream before.
498 if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) {
499 streaminfos_[stream_idx].key_frame_request = true;
500 }
501 streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0;
502
503 // Slice the temporal layers out of the full allocation and pass it on to
504 // the encoder handling the current simulcast stream.
505 RateControlParameters stream_parameters = parameters;
506 stream_parameters.bitrate = VideoBitrateAllocation();
507 for (int i = 0; i < kMaxTemporalStreams; ++i) {
508 if (parameters.bitrate.HasBitrate(stream_idx, i)) {
509 stream_parameters.bitrate.SetBitrate(
510 0, i, parameters.bitrate.GetBitrate(stream_idx, i));
511 }
512 }
513
514 // Assign link allocation proportionally to spatial layer allocation.
515 if (!parameters.bandwidth_allocation.IsZero() &&
516 parameters.bitrate.get_sum_bps() > 0) {
517 stream_parameters.bandwidth_allocation =
518 DataRate::BitsPerSec((parameters.bandwidth_allocation.bps() *
519 stream_parameters.bitrate.get_sum_bps()) /
520 parameters.bitrate.get_sum_bps());
521 // Make sure we don't allocate bandwidth lower than target bitrate.
522 if (stream_parameters.bandwidth_allocation.bps() <
523 stream_parameters.bitrate.get_sum_bps()) {
524 stream_parameters.bandwidth_allocation =
525 DataRate::BitsPerSec(stream_parameters.bitrate.get_sum_bps());
526 }
527 }
528
529 stream_parameters.framerate_fps = std::min<double>(
530 parameters.framerate_fps,
531 streaminfos_[stream_idx].framerate_controller->GetTargetRate());
532
533 streaminfos_[stream_idx].encoder->SetRates(stream_parameters);
534 }
535 }
536
OnPacketLossRateUpdate(float packet_loss_rate)537 void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
538 for (StreamInfo& info : streaminfos_) {
539 info.encoder->OnPacketLossRateUpdate(packet_loss_rate);
540 }
541 }
542
OnRttUpdate(int64_t rtt_ms)543 void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
544 for (StreamInfo& info : streaminfos_) {
545 info.encoder->OnRttUpdate(rtt_ms);
546 }
547 }
548
OnLossNotification(const LossNotification & loss_notification)549 void SimulcastEncoderAdapter::OnLossNotification(
550 const LossNotification& loss_notification) {
551 for (StreamInfo& info : streaminfos_) {
552 info.encoder->OnLossNotification(loss_notification);
553 }
554 }
555
556 // TODO(brandtr): Add task checker to this member function, when all encoder
557 // callbacks are coming in on the encoder queue.
OnEncodedImage(size_t stream_idx,const EncodedImage & encodedImage,const CodecSpecificInfo * codecSpecificInfo)558 EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
559 size_t stream_idx,
560 const EncodedImage& encodedImage,
561 const CodecSpecificInfo* codecSpecificInfo) {
562 EncodedImage stream_image(encodedImage);
563 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
564
565 stream_image.SetSpatialIndex(stream_idx);
566
567 return encoded_complete_callback_->OnEncodedImage(stream_image,
568 &stream_codec_specific);
569 }
570
PopulateStreamCodec(const webrtc::VideoCodec & inst,int stream_index,uint32_t start_bitrate_kbps,StreamResolution stream_resolution,webrtc::VideoCodec * stream_codec)571 void SimulcastEncoderAdapter::PopulateStreamCodec(
572 const webrtc::VideoCodec& inst,
573 int stream_index,
574 uint32_t start_bitrate_kbps,
575 StreamResolution stream_resolution,
576 webrtc::VideoCodec* stream_codec) {
577 *stream_codec = inst;
578
579 // Stream specific settings.
580 stream_codec->numberOfSimulcastStreams = 0;
581 stream_codec->width = inst.simulcastStream[stream_index].width;
582 stream_codec->height = inst.simulcastStream[stream_index].height;
583 stream_codec->maxBitrate = inst.simulcastStream[stream_index].maxBitrate;
584 stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
585 stream_codec->maxFramerate = inst.simulcastStream[stream_index].maxFramerate;
586 stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
587 stream_codec->active = inst.simulcastStream[stream_index].active;
588 // Settings that are based on stream/resolution.
589 if (stream_resolution == StreamResolution::LOWEST) {
590 // Settings for lowest spatial resolutions.
591 if (inst.mode == VideoCodecMode::kScreensharing) {
592 if (experimental_boosted_screenshare_qp_) {
593 stream_codec->qpMax = *experimental_boosted_screenshare_qp_;
594 }
595 } else if (boost_base_layer_quality_) {
596 stream_codec->qpMax = kLowestResMaxQp;
597 }
598 }
599 if (inst.codecType == webrtc::kVideoCodecVP8) {
600 stream_codec->VP8()->numberOfTemporalLayers =
601 inst.simulcastStream[stream_index].numberOfTemporalLayers;
602 if (stream_resolution != StreamResolution::HIGHEST) {
603 // For resolutions below CIF, set the codec |complexity| parameter to
604 // kComplexityHigher, which maps to cpu_used = -4.
605 int pixels_per_frame = stream_codec->width * stream_codec->height;
606 if (pixels_per_frame < 352 * 288) {
607 stream_codec->VP8()->complexity =
608 webrtc::VideoCodecComplexity::kComplexityHigher;
609 }
610 // Turn off denoising for all streams but the highest resolution.
611 stream_codec->VP8()->denoisingOn = false;
612 }
613 } else if (inst.codecType == webrtc::kVideoCodecH264) {
614 stream_codec->H264()->numberOfTemporalLayers =
615 inst.simulcastStream[stream_index].numberOfTemporalLayers;
616 }
617 // TODO(ronghuawu): what to do with targetBitrate.
618
619 stream_codec->startBitrate = start_bitrate_kbps;
620
621 // Legacy screenshare mode is only enabled for the first simulcast layer
622 stream_codec->legacy_conference_mode =
623 inst.legacy_conference_mode && stream_index == 0;
624 }
625
Initialized() const626 bool SimulcastEncoderAdapter::Initialized() const {
627 return rtc::AtomicOps::AcquireLoad(&inited_) == 1;
628 }
629
DestroyStoredEncoders()630 void SimulcastEncoderAdapter::DestroyStoredEncoders() {
631 while (!stored_encoders_.empty()) {
632 stored_encoders_.pop();
633 }
634 }
635
GetEncoderInfo() const636 VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
637 if (streaminfos_.size() == 1) {
638 // Not using simulcast adapting functionality, just pass through.
639 return streaminfos_[0].encoder->GetEncoderInfo();
640 }
641
642 VideoEncoder::EncoderInfo encoder_info;
643 encoder_info.implementation_name = "SimulcastEncoderAdapter";
644 encoder_info.requested_resolution_alignment = 1;
645 encoder_info.apply_alignment_to_all_simulcast_layers = false;
646 encoder_info.supports_native_handle = true;
647 encoder_info.scaling_settings.thresholds = absl::nullopt;
648 if (streaminfos_.empty()) {
649 return encoder_info;
650 }
651
652 encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
653 int num_active_streams = NumActiveStreams(codec_);
654
655 for (size_t i = 0; i < streaminfos_.size(); ++i) {
656 VideoEncoder::EncoderInfo encoder_impl_info =
657 streaminfos_[i].encoder->GetEncoderInfo();
658
659 if (i == 0) {
660 // Encoder name indicates names of all sub-encoders.
661 encoder_info.implementation_name += " (";
662 encoder_info.implementation_name += encoder_impl_info.implementation_name;
663
664 encoder_info.supports_native_handle =
665 encoder_impl_info.supports_native_handle;
666 encoder_info.has_trusted_rate_controller =
667 encoder_impl_info.has_trusted_rate_controller;
668 encoder_info.is_hardware_accelerated =
669 encoder_impl_info.is_hardware_accelerated;
670 encoder_info.has_internal_source = encoder_impl_info.has_internal_source;
671 } else {
672 encoder_info.implementation_name += ", ";
673 encoder_info.implementation_name += encoder_impl_info.implementation_name;
674
675 // Native handle supported if any encoder supports it.
676 encoder_info.supports_native_handle |=
677 encoder_impl_info.supports_native_handle;
678
679 // Trusted rate controller only if all encoders have it.
680 encoder_info.has_trusted_rate_controller &=
681 encoder_impl_info.has_trusted_rate_controller;
682
683 // Uses hardware support if any of the encoders uses it.
684 // For example, if we are having issues with down-scaling due to
685 // pipelining delay in HW encoders we need higher encoder usage
686 // thresholds in CPU adaptation.
687 encoder_info.is_hardware_accelerated |=
688 encoder_impl_info.is_hardware_accelerated;
689
690 // Has internal source only if all encoders have it.
691 encoder_info.has_internal_source &= encoder_impl_info.has_internal_source;
692 }
693 encoder_info.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
694 encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
695 encoder_info.requested_resolution_alignment,
696 encoder_impl_info.requested_resolution_alignment);
697 if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
698 encoder_info.apply_alignment_to_all_simulcast_layers = true;
699 }
700 if (num_active_streams == 1 && codec_.simulcastStream[i].active) {
701 encoder_info.scaling_settings = encoder_impl_info.scaling_settings;
702 }
703 }
704 encoder_info.implementation_name += ")";
705
706 return encoder_info;
707 }
708
709 } // namespace webrtc
710