1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11
12 #include <algorithm> // std::max
13
14 #include "common_types.h" // NOLINT(build/include)
15 #include "common_video/include/video_bitrate_allocator.h"
16 #include "common_video/libyuv/include/webrtc_libyuv.h"
17 #include "modules/video_coding/codecs/vp8/temporal_layers.h"
18 #include "modules/video_coding/encoded_frame.h"
19 #include "modules/video_coding/include/video_codec_interface.h"
20 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
21 #include "modules/video_coding/utility/quality_scaler.h"
22 #include "modules/video_coding/video_coding_impl.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/logging.h"
25 #include "system_wrappers/include/clock.h"
26
27 namespace webrtc {
28 namespace vcm {
29
VideoSender(Clock * clock,EncodedImageCallback * post_encode_callback)30 VideoSender::VideoSender(Clock* clock,
31 EncodedImageCallback* post_encode_callback)
32 : _encoder(nullptr),
33 _mediaOpt(clock),
34 _encodedFrameCallback(post_encode_callback, &_mediaOpt),
35 post_encode_callback_(post_encode_callback),
36 _codecDataBase(&_encodedFrameCallback),
37 frame_dropper_enabled_(true),
38 current_codec_(),
39 encoder_params_({BitrateAllocation(), 0, 0, 0}),
40 encoder_has_internal_source_(false),
41 next_frame_types_(1, kVideoFrameDelta) {
42 _mediaOpt.Reset();
43 // Allow VideoSender to be created on one thread but used on another, post
44 // construction. This is currently how this class is being used by at least
45 // one external project (diffractor).
46 sequenced_checker_.Detach();
47 }
48
~VideoSender()49 VideoSender::~VideoSender() {}
50
51 // Register the send codec to be used.
RegisterSendCodec(const VideoCodec * sendCodec,uint32_t numberOfCores,uint32_t maxPayloadSize)52 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
53 uint32_t numberOfCores,
54 uint32_t maxPayloadSize) {
55 RTC_DCHECK(sequenced_checker_.CalledSequentially());
56 rtc::CritScope lock(&encoder_crit_);
57 if (sendCodec == nullptr) {
58 return VCM_PARAMETER_ERROR;
59 }
60
61 bool ret =
62 _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize);
63
64 // Update encoder regardless of result to make sure that we're not holding on
65 // to a deleted instance.
66 _encoder = _codecDataBase.GetEncoder();
67 // Cache the current codec here so they can be fetched from this thread
68 // without requiring the _sendCritSect lock.
69 current_codec_ = *sendCodec;
70
71 if (!ret) {
72 RTC_LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '"
73 << sendCodec->plName << "'.";
74 return VCM_CODEC_ERROR;
75 }
76
77 // SetSendCodec succeeded, _encoder should be set.
78 RTC_DCHECK(_encoder);
79
80 int numLayers;
81 if (sendCodec->codecType == kVideoCodecVP8) {
82 numLayers = sendCodec->VP8().numberOfTemporalLayers;
83 } else if (sendCodec->codecType == kVideoCodecVP9) {
84 numLayers = sendCodec->VP9().numberOfTemporalLayers;
85 } else if (sendCodec->codecType == kVideoCodecGeneric &&
86 sendCodec->numberOfSimulcastStreams > 0) {
87 // This is mainly for unit testing, disabling frame dropping.
88 // TODO(sprang): Add a better way to disable frame dropping.
89 numLayers = sendCodec->simulcastStream[0].numberOfTemporalLayers;
90 } else {
91 numLayers = 1;
92 }
93
94 // If we have screensharing and we have layers, we disable frame dropper.
95 bool disable_frame_dropper =
96 numLayers > 1 && sendCodec->mode == kScreensharing;
97 if (disable_frame_dropper) {
98 _mediaOpt.EnableFrameDropper(false);
99 } else if (frame_dropper_enabled_) {
100 _mediaOpt.EnableFrameDropper(true);
101 }
102
103 {
104 rtc::CritScope cs(¶ms_crit_);
105 next_frame_types_.clear();
106 next_frame_types_.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
107 kVideoFrameKey);
108 // Cache InternalSource() to have this available from IntraFrameRequest()
109 // without having to acquire encoder_crit_ (avoid blocking on encoder use).
110 encoder_has_internal_source_ = _encoder->InternalSource();
111 }
112
113 RTC_LOG(LS_VERBOSE) << " max bitrate " << sendCodec->maxBitrate
114 << " start bitrate " << sendCodec->startBitrate
115 << " max frame rate " << sendCodec->maxFramerate
116 << " max payload size " << maxPayloadSize;
117 _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000,
118 sendCodec->startBitrate * 1000,
119 sendCodec->maxFramerate);
120 return VCM_OK;
121 }
122
123 // Register an external decoder object.
124 // This can not be used together with external decoder callbacks.
RegisterExternalEncoder(VideoEncoder * externalEncoder,uint8_t payloadType,bool internalSource)125 void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
126 uint8_t payloadType,
127 bool internalSource /*= false*/) {
128 RTC_DCHECK(sequenced_checker_.CalledSequentially());
129
130 rtc::CritScope lock(&encoder_crit_);
131
132 if (externalEncoder == nullptr) {
133 bool wasSendCodec = false;
134 RTC_CHECK(
135 _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec));
136 if (wasSendCodec) {
137 // Make sure the VCM doesn't use the de-registered codec
138 rtc::CritScope params_lock(¶ms_crit_);
139 _encoder = nullptr;
140 encoder_has_internal_source_ = false;
141 }
142 return;
143 }
144 _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType,
145 internalSource);
146 }
147
148 // Get encode bitrate
Bitrate(unsigned int * bitrate) const149 int VideoSender::Bitrate(unsigned int* bitrate) const {
150 RTC_DCHECK(sequenced_checker_.CalledSequentially());
151 // Since we're running on the thread that's the only thread known to modify
152 // the value of _encoder, we don't need to grab the lock here.
153
154 if (!_encoder)
155 return VCM_UNINITIALIZED;
156 *bitrate = _encoder->GetEncoderParameters().target_bitrate.get_sum_bps();
157 return 0;
158 }
159
160 // Get encode frame rate
FrameRate(unsigned int * framerate) const161 int VideoSender::FrameRate(unsigned int* framerate) const {
162 RTC_DCHECK(sequenced_checker_.CalledSequentially());
163 // Since we're running on the thread that's the only thread known to modify
164 // the value of _encoder, we don't need to grab the lock here.
165
166 if (!_encoder)
167 return VCM_UNINITIALIZED;
168
169 *framerate = _encoder->GetEncoderParameters().input_frame_rate;
170 return 0;
171 }
172
UpdateEncoderParameters(const EncoderParameters & params,VideoBitrateAllocator * bitrate_allocator,uint32_t target_bitrate_bps)173 EncoderParameters VideoSender::UpdateEncoderParameters(
174 const EncoderParameters& params,
175 VideoBitrateAllocator* bitrate_allocator,
176 uint32_t target_bitrate_bps) {
177 uint32_t video_target_rate_bps = _mediaOpt.SetTargetRates(target_bitrate_bps);
178 uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
179 if (input_frame_rate == 0 || input_frame_rate > current_codec_.maxFramerate)
180 input_frame_rate = current_codec_.maxFramerate;
181
182 BitrateAllocation bitrate_allocation;
183 // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
184 // might cap the bitrate to the min bitrate configured.
185 if (target_bitrate_bps > 0) {
186 if (bitrate_allocator) {
187 bitrate_allocation = bitrate_allocator->GetAllocation(
188 video_target_rate_bps, input_frame_rate);
189 } else {
190 DefaultVideoBitrateAllocator default_allocator(current_codec_);
191 bitrate_allocation = default_allocator.GetAllocation(
192 video_target_rate_bps, input_frame_rate);
193 }
194 }
195 EncoderParameters new_encoder_params = {bitrate_allocation, params.loss_rate,
196 params.rtt, input_frame_rate};
197 return new_encoder_params;
198 }
199
UpdateChannelParemeters(VideoBitrateAllocator * bitrate_allocator,VideoBitrateAllocationObserver * bitrate_updated_callback)200 void VideoSender::UpdateChannelParemeters(
201 VideoBitrateAllocator* bitrate_allocator,
202 VideoBitrateAllocationObserver* bitrate_updated_callback) {
203 BitrateAllocation target_rate;
204 {
205 rtc::CritScope cs(¶ms_crit_);
206 encoder_params_ =
207 UpdateEncoderParameters(encoder_params_, bitrate_allocator,
208 encoder_params_.target_bitrate.get_sum_bps());
209 target_rate = encoder_params_.target_bitrate;
210 }
211 if (bitrate_updated_callback && target_rate.get_sum_bps() > 0)
212 bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate);
213 }
214
SetChannelParameters(uint32_t target_bitrate_bps,uint8_t loss_rate,int64_t rtt,VideoBitrateAllocator * bitrate_allocator,VideoBitrateAllocationObserver * bitrate_updated_callback)215 int32_t VideoSender::SetChannelParameters(
216 uint32_t target_bitrate_bps,
217 uint8_t loss_rate,
218 int64_t rtt,
219 VideoBitrateAllocator* bitrate_allocator,
220 VideoBitrateAllocationObserver* bitrate_updated_callback) {
221 EncoderParameters encoder_params;
222 encoder_params.loss_rate = loss_rate;
223 encoder_params.rtt = rtt;
224 encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator,
225 target_bitrate_bps);
226 if (bitrate_updated_callback && target_bitrate_bps > 0) {
227 bitrate_updated_callback->OnBitrateAllocationUpdated(
228 encoder_params.target_bitrate);
229 }
230
231 bool encoder_has_internal_source;
232 {
233 rtc::CritScope cs(¶ms_crit_);
234 encoder_params_ = encoder_params;
235 encoder_has_internal_source = encoder_has_internal_source_;
236 }
237
238 // For encoders with internal sources, we need to tell the encoder directly,
239 // instead of waiting for an AddVideoFrame that will never come (internal
240 // source encoders don't get input frames).
241 if (encoder_has_internal_source) {
242 rtc::CritScope cs(&encoder_crit_);
243 if (_encoder) {
244 SetEncoderParameters(encoder_params, encoder_has_internal_source);
245 }
246 }
247
248 return VCM_OK;
249 }
250
SetEncoderParameters(EncoderParameters params,bool has_internal_source)251 void VideoSender::SetEncoderParameters(EncoderParameters params,
252 bool has_internal_source) {
253 // |target_bitrate == 0 | means that the network is down or the send pacer is
254 // full. We currently only report this if the encoder has an internal source.
255 // If the encoder does not have an internal source, higher levels are expected
256 // to not call AddVideoFrame. We do this since its unclear how current
257 // encoder implementations behave when given a zero target bitrate.
258 // TODO(perkj): Make sure all known encoder implementations handle zero
259 // target bitrate and remove this check.
260 if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0)
261 return;
262
263 if (params.input_frame_rate == 0) {
264 // No frame rate estimate available, use default.
265 params.input_frame_rate = current_codec_.maxFramerate;
266 }
267 if (_encoder != nullptr)
268 _encoder->SetEncoderParameters(params);
269 }
270
271 // Deprecated:
272 // TODO(perkj): Remove once no projects call this method. It currently do
273 // nothing.
RegisterProtectionCallback(VCMProtectionCallback * protection_callback)274 int32_t VideoSender::RegisterProtectionCallback(
275 VCMProtectionCallback* protection_callback) {
276 // Deprecated:
277 // TODO(perkj): Remove once no projects call this method. It currently do
278 // nothing.
279 return VCM_OK;
280 }
281
282 // Add one raw video frame to the encoder, blocking.
AddVideoFrame(const VideoFrame & videoFrame,const CodecSpecificInfo * codecSpecificInfo)283 int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
284 const CodecSpecificInfo* codecSpecificInfo) {
285 EncoderParameters encoder_params;
286 std::vector<FrameType> next_frame_types;
287 bool encoder_has_internal_source = false;
288 {
289 rtc::CritScope lock(¶ms_crit_);
290 encoder_params = encoder_params_;
291 next_frame_types = next_frame_types_;
292 encoder_has_internal_source = encoder_has_internal_source_;
293 }
294 rtc::CritScope lock(&encoder_crit_);
295 if (_encoder == nullptr)
296 return VCM_UNINITIALIZED;
297 SetEncoderParameters(encoder_params, encoder_has_internal_source);
298 if (_mediaOpt.DropFrame()) {
299 RTC_LOG(LS_VERBOSE) << "Drop Frame "
300 << "target bitrate "
301 << encoder_params.target_bitrate.get_sum_bps()
302 << " loss rate " << encoder_params.loss_rate << " rtt "
303 << encoder_params.rtt << " input frame rate "
304 << encoder_params.input_frame_rate;
305 post_encode_callback_->OnDroppedFrame(
306 EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
307 return VCM_OK;
308 }
309 // TODO(pbos): Make sure setting send codec is synchronized with video
310 // processing so frame size always matches.
311 if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
312 videoFrame.height())) {
313 RTC_LOG(LS_ERROR)
314 << "Incoming frame doesn't match set resolution. Dropping.";
315 return VCM_PARAMETER_ERROR;
316 }
317 VideoFrame converted_frame = videoFrame;
318 const VideoFrameBuffer::Type buffer_type =
319 converted_frame.video_frame_buffer()->type();
320 const bool is_buffer_type_supported =
321 buffer_type == VideoFrameBuffer::Type::kI420 ||
322 (buffer_type == VideoFrameBuffer::Type::kNative &&
323 _encoder->SupportsNativeHandle());
324 if (!is_buffer_type_supported) {
325 // This module only supports software encoding.
326 // TODO(pbos): Offload conversion from the encoder thread.
327 rtc::scoped_refptr<I420BufferInterface> converted_buffer(
328 converted_frame.video_frame_buffer()->ToI420());
329
330 if (!converted_buffer) {
331 RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
332 return VCM_PARAMETER_ERROR;
333 }
334 converted_frame = VideoFrame(converted_buffer,
335 converted_frame.timestamp(),
336 converted_frame.render_time_ms(),
337 converted_frame.rotation());
338 }
339 int32_t ret =
340 _encoder->Encode(converted_frame, codecSpecificInfo, next_frame_types);
341 if (ret < 0) {
342 RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
343 return ret;
344 }
345
346 {
347 rtc::CritScope lock(¶ms_crit_);
348 // Change all keyframe requests to encode delta frames the next time.
349 for (size_t i = 0; i < next_frame_types_.size(); ++i) {
350 // Check for equality (same requested as before encoding) to not
351 // accidentally drop a keyframe request while encoding.
352 if (next_frame_types[i] == next_frame_types_[i])
353 next_frame_types_[i] = kVideoFrameDelta;
354 }
355 }
356 return VCM_OK;
357 }
358
IntraFrameRequest(size_t stream_index)359 int32_t VideoSender::IntraFrameRequest(size_t stream_index) {
360 {
361 rtc::CritScope lock(¶ms_crit_);
362 if (stream_index >= next_frame_types_.size()) {
363 return -1;
364 }
365 next_frame_types_[stream_index] = kVideoFrameKey;
366 if (!encoder_has_internal_source_)
367 return VCM_OK;
368 }
369 // TODO(pbos): Remove when InternalSource() is gone. Both locks have to be
370 // held here for internal consistency, since _encoder could be removed while
371 // not holding encoder_crit_. Checks have to be performed again since
372 // params_crit_ was dropped to not cause lock-order inversions with
373 // encoder_crit_.
374 rtc::CritScope lock(&encoder_crit_);
375 rtc::CritScope params_lock(¶ms_crit_);
376 if (stream_index >= next_frame_types_.size())
377 return -1;
378 if (_encoder != nullptr && _encoder->InternalSource()) {
379 // Try to request the frame if we have an external encoder with
380 // internal source since AddVideoFrame never will be called.
381 if (_encoder->RequestFrame(next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) {
382 // Try to remove just-performed keyframe request, if stream still exists.
383 next_frame_types_[stream_index] = kVideoFrameDelta;
384 }
385 }
386 return VCM_OK;
387 }
388
EnableFrameDropper(bool enable)389 int32_t VideoSender::EnableFrameDropper(bool enable) {
390 rtc::CritScope lock(&encoder_crit_);
391 frame_dropper_enabled_ = enable;
392 _mediaOpt.EnableFrameDropper(enable);
393 return VCM_OK;
394 }
395
396 } // namespace vcm
397 } // namespace webrtc
398