1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "webrtc/video/video_send_stream.h"
11
12 #include <algorithm>
13 #include <cmath>
14 #include <sstream>
15 #include <string>
16 #include <utility>
17 #include <vector>
18
19 #include "webrtc/common_types.h"
20 #include "webrtc/common_video/include/video_bitrate_allocator.h"
21 #include "webrtc/base/checks.h"
22 #include "webrtc/base/file.h"
23 #include "webrtc/base/logging.h"
24 #include "webrtc/base/trace_event.h"
25 #include "webrtc/base/weak_ptr.h"
26 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
27 #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
28 #include "webrtc/modules/pacing/packet_router.h"
29 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
30 #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h"
31 #include "webrtc/modules/utility/include/process_thread.h"
32 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
33 #include "webrtc/system_wrappers/include/field_trial.h"
34 #include "webrtc/video/call_stats.h"
35 #include "webrtc/video/vie_remb.h"
36 #include "webrtc/video_send_stream.h"
37
38 namespace webrtc {
39
40 static const int kMinSendSidePacketHistorySize = 600;
41 namespace {
42
43 // We don't do MTU discovery, so assume that we have the standard ethernet MTU.
44 const size_t kPathMTU = 1500;
45
CreateRtpRtcpModules(Transport * outgoing_transport,RtcpIntraFrameObserver * intra_frame_callback,RtcpBandwidthObserver * bandwidth_callback,TransportFeedbackObserver * transport_feedback_callback,RtcpRttStats * rtt_stats,RtpPacketSender * paced_sender,TransportSequenceNumberAllocator * transport_sequence_number_allocator,FlexfecSender * flexfec_sender,SendStatisticsProxy * stats_proxy,SendDelayStats * send_delay_stats,RtcEventLog * event_log,RateLimiter * retransmission_rate_limiter,OverheadObserver * overhead_observer,size_t num_modules)46 std::vector<RtpRtcp*> CreateRtpRtcpModules(
47 Transport* outgoing_transport,
48 RtcpIntraFrameObserver* intra_frame_callback,
49 RtcpBandwidthObserver* bandwidth_callback,
50 TransportFeedbackObserver* transport_feedback_callback,
51 RtcpRttStats* rtt_stats,
52 RtpPacketSender* paced_sender,
53 TransportSequenceNumberAllocator* transport_sequence_number_allocator,
54 FlexfecSender* flexfec_sender,
55 SendStatisticsProxy* stats_proxy,
56 SendDelayStats* send_delay_stats,
57 RtcEventLog* event_log,
58 RateLimiter* retransmission_rate_limiter,
59 OverheadObserver* overhead_observer,
60 size_t num_modules) {
61 RTC_DCHECK_GT(num_modules, 0);
62 RtpRtcp::Configuration configuration;
63 ReceiveStatistics* null_receive_statistics = configuration.receive_statistics;
64 configuration.audio = false;
65 configuration.receiver_only = false;
66 configuration.flexfec_sender = flexfec_sender;
67 configuration.receive_statistics = null_receive_statistics;
68 configuration.outgoing_transport = outgoing_transport;
69 configuration.intra_frame_callback = intra_frame_callback;
70 configuration.bandwidth_callback = bandwidth_callback;
71 configuration.transport_feedback_callback = transport_feedback_callback;
72 configuration.rtt_stats = rtt_stats;
73 configuration.rtcp_packet_type_counter_observer = stats_proxy;
74 configuration.paced_sender = paced_sender;
75 configuration.transport_sequence_number_allocator =
76 transport_sequence_number_allocator;
77 configuration.send_bitrate_observer = stats_proxy;
78 configuration.send_frame_count_observer = stats_proxy;
79 configuration.send_side_delay_observer = stats_proxy;
80 configuration.send_packet_observer = send_delay_stats;
81 configuration.event_log = event_log;
82 configuration.retransmission_rate_limiter = retransmission_rate_limiter;
83 configuration.overhead_observer = overhead_observer;
84 std::vector<RtpRtcp*> modules;
85 for (size_t i = 0; i < num_modules; ++i) {
86 RtpRtcp* rtp_rtcp = RtpRtcp::CreateRtpRtcp(configuration);
87 rtp_rtcp->SetSendingStatus(false);
88 rtp_rtcp->SetSendingMediaStatus(false);
89 rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
90 modules.push_back(rtp_rtcp);
91 }
92 return modules;
93 }
94
95 // TODO(brandtr): Update this function when we support multistream protection.
MaybeCreateFlexfecSender(const VideoSendStream::Config & config)96 std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
97 const VideoSendStream::Config& config) {
98 if (config.rtp.flexfec.payload_type < 0) {
99 return nullptr;
100 }
101 RTC_DCHECK_GE(config.rtp.flexfec.payload_type, 0);
102 RTC_DCHECK_LE(config.rtp.flexfec.payload_type, 127);
103 if (config.rtp.flexfec.ssrc == 0) {
104 LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
105 "Therefore disabling FlexFEC.";
106 return nullptr;
107 }
108 if (config.rtp.flexfec.protected_media_ssrcs.empty()) {
109 LOG(LS_WARNING) << "FlexFEC is enabled, but no protected media SSRC given. "
110 "Therefore disabling FlexFEC.";
111 return nullptr;
112 }
113
114 if (config.rtp.ssrcs.size() > 1) {
115 LOG(LS_WARNING) << "Both FlexFEC and simulcast are enabled. This "
116 "combination is however not supported by our current "
117 "FlexFEC implementation. Therefore disabling FlexFEC.";
118 return nullptr;
119 }
120
121 if (config.rtp.flexfec.protected_media_ssrcs.size() > 1) {
122 LOG(LS_WARNING)
123 << "The supplied FlexfecConfig contained multiple protected "
124 "media streams, but our implementation currently only "
125 "supports protecting a single media stream. "
126 "To avoid confusion, disabling FlexFEC completely.";
127 return nullptr;
128 }
129
130 RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size());
131 return std::unique_ptr<FlexfecSender>(new FlexfecSender(
132 config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc,
133 config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.extensions,
134 RTPSender::FecExtensionSizes(), Clock::GetRealTimeClock()));
135 }
136
137 } // namespace
138
139 std::string
ToString() const140 VideoSendStream::Config::EncoderSettings::ToString() const {
141 std::stringstream ss;
142 ss << "{payload_name: " << payload_name;
143 ss << ", payload_type: " << payload_type;
144 ss << ", encoder: " << (encoder ? "(VideoEncoder)" : "nullptr");
145 ss << '}';
146 return ss.str();
147 }
148
ToString() const149 std::string VideoSendStream::Config::Rtp::Rtx::ToString()
150 const {
151 std::stringstream ss;
152 ss << "{ssrcs: [";
153 for (size_t i = 0; i < ssrcs.size(); ++i) {
154 ss << ssrcs[i];
155 if (i != ssrcs.size() - 1)
156 ss << ", ";
157 }
158 ss << ']';
159
160 ss << ", payload_type: " << payload_type;
161 ss << '}';
162 return ss.str();
163 }
164
ToString() const165 std::string VideoSendStream::Config::Rtp::ToString() const {
166 std::stringstream ss;
167 ss << "{ssrcs: [";
168 for (size_t i = 0; i < ssrcs.size(); ++i) {
169 ss << ssrcs[i];
170 if (i != ssrcs.size() - 1)
171 ss << ", ";
172 }
173 ss << ']';
174 ss << ", rtcp_mode: "
175 << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound"
176 : "RtcpMode::kReducedSize");
177 ss << ", max_packet_size: " << max_packet_size;
178 ss << ", extensions: [";
179 for (size_t i = 0; i < extensions.size(); ++i) {
180 ss << extensions[i].ToString();
181 if (i != extensions.size() - 1)
182 ss << ", ";
183 }
184 ss << ']';
185
186 ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
187 ss << ", ulpfec: " << ulpfec.ToString();
188
189 ss << ", flexfec: {payload_type: " << flexfec.payload_type;
190 ss << ", ssrc: " << flexfec.ssrc;
191 ss << ", protected_media_ssrcs: [";
192 for (size_t i = 0; i < flexfec.protected_media_ssrcs.size(); ++i) {
193 ss << flexfec.protected_media_ssrcs[i];
194 if (i != flexfec.protected_media_ssrcs.size() - 1)
195 ss << ", ";
196 }
197 ss << ']';
198
199 ss << ", rtx: " << rtx.ToString();
200 ss << ", c_name: " << c_name;
201 ss << '}';
202 return ss.str();
203 }
204
ToString() const205 std::string VideoSendStream::Config::ToString() const {
206 std::stringstream ss;
207 ss << "{encoder_settings: " << encoder_settings.ToString();
208 ss << ", rtp: " << rtp.ToString();
209 ss << ", pre_encode_callback: "
210 << (pre_encode_callback ? "(I420FrameCallback)" : "nullptr");
211 ss << ", post_encode_callback: "
212 << (post_encode_callback ? "(EncodedFrameObserver)" : "nullptr");
213 ss << ", render_delay_ms: " << render_delay_ms;
214 ss << ", target_delay_ms: " << target_delay_ms;
215 ss << ", suspend_below_min_bitrate: " << (suspend_below_min_bitrate ? "on"
216 : "off");
217 ss << '}';
218 return ss.str();
219 }
220
ToString(int64_t time_ms) const221 std::string VideoSendStream::Stats::ToString(int64_t time_ms) const {
222 std::stringstream ss;
223 ss << "VideoSendStream stats: " << time_ms << ", {";
224 ss << "input_fps: " << input_frame_rate << ", ";
225 ss << "encode_fps: " << encode_frame_rate << ", ";
226 ss << "encode_ms: " << avg_encode_time_ms << ", ";
227 ss << "encode_usage_perc: " << encode_usage_percent << ", ";
228 ss << "target_bps: " << target_media_bitrate_bps << ", ";
229 ss << "media_bps: " << media_bitrate_bps << ", ";
230 ss << "preferred_media_bitrate_bps: " << preferred_media_bitrate_bps << ", ";
231 ss << "suspended: " << (suspended ? "true" : "false") << ", ";
232 ss << "bw_adapted: " << (bw_limited_resolution ? "true" : "false");
233 ss << '}';
234 for (const auto& substream : substreams) {
235 if (!substream.second.is_rtx && !substream.second.is_flexfec) {
236 ss << " {ssrc: " << substream.first << ", ";
237 ss << substream.second.ToString();
238 ss << '}';
239 }
240 }
241 return ss.str();
242 }
243
ToString() const244 std::string VideoSendStream::StreamStats::ToString() const {
245 std::stringstream ss;
246 ss << "width: " << width << ", ";
247 ss << "height: " << height << ", ";
248 ss << "key: " << frame_counts.key_frames << ", ";
249 ss << "delta: " << frame_counts.delta_frames << ", ";
250 ss << "total_bps: " << total_bitrate_bps << ", ";
251 ss << "retransmit_bps: " << retransmit_bitrate_bps << ", ";
252 ss << "avg_delay_ms: " << avg_delay_ms << ", ";
253 ss << "max_delay_ms: " << max_delay_ms << ", ";
254 ss << "cum_loss: " << rtcp_stats.cumulative_lost << ", ";
255 ss << "max_ext_seq: " << rtcp_stats.extended_max_sequence_number << ", ";
256 ss << "nack: " << rtcp_packet_type_counts.nack_packets << ", ";
257 ss << "fir: " << rtcp_packet_type_counts.fir_packets << ", ";
258 ss << "pli: " << rtcp_packet_type_counts.pli_packets;
259 return ss.str();
260 }
261
262 namespace {
263
PayloadTypeSupportsSkippingFecPackets(const std::string & payload_name)264 bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
265 if (payload_name == "VP8" || payload_name == "VP9")
266 return true;
267 RTC_DCHECK(payload_name == "H264" || payload_name == "FAKE")
268 << "unknown payload_name " << payload_name;
269 return false;
270 }
271
CalculateMaxPadBitrateBps(std::vector<VideoStream> streams,int min_transmit_bitrate_bps,bool pad_to_min_bitrate)272 int CalculateMaxPadBitrateBps(std::vector<VideoStream> streams,
273 int min_transmit_bitrate_bps,
274 bool pad_to_min_bitrate) {
275 int pad_up_to_bitrate_bps = 0;
276 // Calculate max padding bitrate for a multi layer codec.
277 if (streams.size() > 1) {
278 // Pad to min bitrate of the highest layer.
279 pad_up_to_bitrate_bps = streams[streams.size() - 1].min_bitrate_bps;
280 // Add target_bitrate_bps of the lower layers.
281 for (size_t i = 0; i < streams.size() - 1; ++i)
282 pad_up_to_bitrate_bps += streams[i].target_bitrate_bps;
283 } else if (pad_to_min_bitrate) {
284 pad_up_to_bitrate_bps = streams[0].min_bitrate_bps;
285 }
286
287 pad_up_to_bitrate_bps =
288 std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps);
289
290 return pad_up_to_bitrate_bps;
291 }
292
293 } // namespace
294
295 namespace internal {
296
297 // VideoSendStreamImpl implements internal::VideoSendStream.
298 // It is created and destroyed on |worker_queue|. The intent is to decrease the
299 // need for locking and to ensure methods are called in sequence.
300 // Public methods except |DeliverRtcp| must be called on |worker_queue|.
301 // DeliverRtcp is called on the libjingle worker thread or a network thread.
302 // An encoder may deliver frames through the EncodedImageCallback on an
303 // arbitrary thread.
304 class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
305 public webrtc::OverheadObserver,
306 public webrtc::VCMProtectionCallback,
307 public ViEEncoder::EncoderSink,
308 public VideoBitrateAllocationObserver {
309 public:
310 VideoSendStreamImpl(SendStatisticsProxy* stats_proxy,
311 rtc::TaskQueue* worker_queue,
312 CallStats* call_stats,
313 CongestionController* congestion_controller,
314 PacketRouter* packet_router,
315 BitrateAllocator* bitrate_allocator,
316 SendDelayStats* send_delay_stats,
317 VieRemb* remb,
318 ViEEncoder* vie_encoder,
319 RtcEventLog* event_log,
320 const VideoSendStream::Config* config,
321 int initial_encoder_max_bitrate,
322 std::map<uint32_t, RtpState> suspended_ssrcs);
323 ~VideoSendStreamImpl() override;
324
325 // RegisterProcessThread register |module_process_thread| with those objects
326 // that use it. Registration has to happen on the thread were
327 // |module_process_thread| was created (libjingle's worker thread).
328 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue,
329 // maybe |worker_queue|.
330 void RegisterProcessThread(ProcessThread* module_process_thread);
331 void DeRegisterProcessThread();
332
333 void SignalNetworkState(NetworkState state);
334 bool DeliverRtcp(const uint8_t* packet, size_t length);
335 void Start();
336 void Stop();
337
338 VideoSendStream::RtpStateMap GetRtpStates() const;
339
340 void EnableEncodedFrameRecording(const std::vector<rtc::PlatformFile>& files,
341 size_t byte_limit);
342
343 void SetTransportOverhead(size_t transport_overhead_per_packet);
344
345 private:
346 class CheckEncoderActivityTask;
347 class EncoderReconfiguredTask;
348
349 // Implements BitrateAllocatorObserver.
350 uint32_t OnBitrateUpdated(uint32_t bitrate_bps,
351 uint8_t fraction_loss,
352 int64_t rtt,
353 int64_t probing_interval_ms) override;
354
355 // Implements webrtc::VCMProtectionCallback.
356 int ProtectionRequest(const FecProtectionParams* delta_params,
357 const FecProtectionParams* key_params,
358 uint32_t* sent_video_rate_bps,
359 uint32_t* sent_nack_rate_bps,
360 uint32_t* sent_fec_rate_bps) override;
361
362 // Implements OverheadObserver.
363 void OnOverheadChanged(size_t overhead_bytes_per_packet) override;
364
365 void OnEncoderConfigurationChanged(std::vector<VideoStream> streams,
366 int min_transmit_bitrate_bps) override;
367
368 // Implements EncodedImageCallback. The implementation routes encoded frames
369 // to the |payload_router_| and |config.pre_encode_callback| if set.
370 // Called on an arbitrary encoder callback thread.
371 EncodedImageCallback::Result OnEncodedImage(
372 const EncodedImage& encoded_image,
373 const CodecSpecificInfo* codec_specific_info,
374 const RTPFragmentationHeader* fragmentation) override;
375
376 // Implements VideoBitrateAllocationObserver.
377 void OnBitrateAllocationUpdated(const BitrateAllocation& allocation) override;
378
379 void ConfigureProtection();
380 void ConfigureSsrcs();
381 void SignalEncoderTimedOut();
382 void SignalEncoderActive();
383
384 SendStatisticsProxy* const stats_proxy_;
385 const VideoSendStream::Config* const config_;
386 std::map<uint32_t, RtpState> suspended_ssrcs_;
387
388 ProcessThread* module_process_thread_;
389 rtc::ThreadChecker module_process_thread_checker_;
390 rtc::TaskQueue* const worker_queue_;
391
392 rtc::CriticalSection encoder_activity_crit_sect_;
393 CheckEncoderActivityTask* check_encoder_activity_task_
394 GUARDED_BY(encoder_activity_crit_sect_);
395
396 CallStats* const call_stats_;
397 CongestionController* const congestion_controller_;
398 PacketRouter* const packet_router_;
399 BitrateAllocator* const bitrate_allocator_;
400 VieRemb* const remb_;
401
402 // TODO(brandtr): Consider moving this to a new FlexfecSendStream class.
403 std::unique_ptr<FlexfecSender> flexfec_sender_;
404
405 rtc::CriticalSection ivf_writers_crit_;
406 std::unique_ptr<IvfFileWriter> file_writers_[kMaxSimulcastStreams] GUARDED_BY(
407 ivf_writers_crit_);
408
409 int max_padding_bitrate_;
410 int encoder_min_bitrate_bps_;
411 uint32_t encoder_max_bitrate_bps_;
412 uint32_t encoder_target_rate_bps_;
413
414 ViEEncoder* const vie_encoder_;
415 EncoderRtcpFeedback encoder_feedback_;
416 ProtectionBitrateCalculator protection_bitrate_calculator_;
417
418 const std::unique_ptr<RtcpBandwidthObserver> bandwidth_observer_;
419 // RtpRtcp modules, declared here as they use other members on construction.
420 const std::vector<RtpRtcp*> rtp_rtcp_modules_;
421 PayloadRouter payload_router_;
422
423 // |weak_ptr_| to our self. This is used since we can not call
424 // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy
425 // an existing WeakPtr.
426 rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_;
427 // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are
428 // invalidated before any other members are destroyed.
429 rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_;
430
431 rtc::CriticalSection overhead_bytes_per_packet_crit_;
432 size_t overhead_bytes_per_packet_ GUARDED_BY(overhead_bytes_per_packet_crit_);
433 size_t transport_overhead_bytes_per_packet_;
434 };
435
436 // TODO(tommi): See if there's a more elegant way to create a task that creates
437 // an object on the correct task queue.
438 class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
439 public:
ConstructionTask(std::unique_ptr<VideoSendStreamImpl> * send_stream,rtc::Event * done_event,SendStatisticsProxy * stats_proxy,ViEEncoder * vie_encoder,ProcessThread * module_process_thread,CallStats * call_stats,CongestionController * congestion_controller,PacketRouter * packet_router,BitrateAllocator * bitrate_allocator,SendDelayStats * send_delay_stats,VieRemb * remb,RtcEventLog * event_log,const VideoSendStream::Config * config,int initial_encoder_max_bitrate,const std::map<uint32_t,RtpState> & suspended_ssrcs)440 ConstructionTask(std::unique_ptr<VideoSendStreamImpl>* send_stream,
441 rtc::Event* done_event,
442 SendStatisticsProxy* stats_proxy,
443 ViEEncoder* vie_encoder,
444 ProcessThread* module_process_thread,
445 CallStats* call_stats,
446 CongestionController* congestion_controller,
447 PacketRouter* packet_router,
448 BitrateAllocator* bitrate_allocator,
449 SendDelayStats* send_delay_stats,
450 VieRemb* remb,
451 RtcEventLog* event_log,
452 const VideoSendStream::Config* config,
453 int initial_encoder_max_bitrate,
454 const std::map<uint32_t, RtpState>& suspended_ssrcs)
455 : send_stream_(send_stream),
456 done_event_(done_event),
457 stats_proxy_(stats_proxy),
458 vie_encoder_(vie_encoder),
459 call_stats_(call_stats),
460 congestion_controller_(congestion_controller),
461 packet_router_(packet_router),
462 bitrate_allocator_(bitrate_allocator),
463 send_delay_stats_(send_delay_stats),
464 remb_(remb),
465 event_log_(event_log),
466 config_(config),
467 initial_encoder_max_bitrate_(initial_encoder_max_bitrate),
468 suspended_ssrcs_(suspended_ssrcs) {}
469
~ConstructionTask()470 ~ConstructionTask() override { done_event_->Set(); }
471
472 private:
Run()473 bool Run() override {
474 send_stream_->reset(new VideoSendStreamImpl(
475 stats_proxy_, rtc::TaskQueue::Current(), call_stats_,
476 congestion_controller_, packet_router_, bitrate_allocator_,
477 send_delay_stats_, remb_, vie_encoder_, event_log_, config_,
478 initial_encoder_max_bitrate_, std::move(suspended_ssrcs_)));
479 return true;
480 }
481
482 std::unique_ptr<VideoSendStreamImpl>* const send_stream_;
483 rtc::Event* const done_event_;
484 SendStatisticsProxy* const stats_proxy_;
485 ViEEncoder* const vie_encoder_;
486 CallStats* const call_stats_;
487 CongestionController* const congestion_controller_;
488 PacketRouter* const packet_router_;
489 BitrateAllocator* const bitrate_allocator_;
490 SendDelayStats* const send_delay_stats_;
491 VieRemb* const remb_;
492 RtcEventLog* const event_log_;
493 const VideoSendStream::Config* config_;
494 int initial_encoder_max_bitrate_;
495 std::map<uint32_t, RtpState> suspended_ssrcs_;
496 };
497
498 class VideoSendStream::DestructAndGetRtpStateTask : public rtc::QueuedTask {
499 public:
DestructAndGetRtpStateTask(VideoSendStream::RtpStateMap * state_map,std::unique_ptr<VideoSendStreamImpl> send_stream,rtc::Event * done_event)500 DestructAndGetRtpStateTask(VideoSendStream::RtpStateMap* state_map,
501 std::unique_ptr<VideoSendStreamImpl> send_stream,
502 rtc::Event* done_event)
503 : state_map_(state_map),
504 send_stream_(std::move(send_stream)),
505 done_event_(done_event) {}
506
~DestructAndGetRtpStateTask()507 ~DestructAndGetRtpStateTask() override { RTC_CHECK(!send_stream_); }
508
509 private:
Run()510 bool Run() override {
511 send_stream_->Stop();
512 *state_map_ = send_stream_->GetRtpStates();
513 send_stream_.reset();
514 done_event_->Set();
515 return true;
516 }
517
518 VideoSendStream::RtpStateMap* state_map_;
519 std::unique_ptr<VideoSendStreamImpl> send_stream_;
520 rtc::Event* done_event_;
521 };
522
523 // CheckEncoderActivityTask is used for tracking when the encoder last produced
524 // and encoded video frame. If the encoder has not produced anything the last
525 // kEncoderTimeOutMs we also want to stop sending padding.
526 class VideoSendStreamImpl::CheckEncoderActivityTask : public rtc::QueuedTask {
527 public:
528 static const int kEncoderTimeOutMs = 2000;
CheckEncoderActivityTask(const rtc::WeakPtr<VideoSendStreamImpl> & send_stream)529 explicit CheckEncoderActivityTask(
530 const rtc::WeakPtr<VideoSendStreamImpl>& send_stream)
531 : activity_(0), send_stream_(std::move(send_stream)), timed_out_(false) {}
532
Stop()533 void Stop() {
534 RTC_CHECK(task_checker_.CalledSequentially());
535 send_stream_.reset();
536 }
537
UpdateEncoderActivity()538 void UpdateEncoderActivity() {
539 // UpdateEncoderActivity is called from VideoSendStreamImpl::Encoded on
540 // whatever thread the real encoder implementation run on. In the case of
541 // hardware encoders, there might be several encoders
542 // running in parallel on different threads.
543 rtc::AtomicOps::ReleaseStore(&activity_, 1);
544 }
545
546 private:
Run()547 bool Run() override {
548 RTC_CHECK(task_checker_.CalledSequentially());
549 if (!send_stream_)
550 return true;
551 if (!rtc::AtomicOps::AcquireLoad(&activity_)) {
552 if (!timed_out_) {
553 send_stream_->SignalEncoderTimedOut();
554 }
555 timed_out_ = true;
556 } else if (timed_out_) {
557 send_stream_->SignalEncoderActive();
558 timed_out_ = false;
559 }
560 rtc::AtomicOps::ReleaseStore(&activity_, 0);
561
562 rtc::TaskQueue::Current()->PostDelayedTask(
563 std::unique_ptr<rtc::QueuedTask>(this), kEncoderTimeOutMs);
564 // Return false to prevent this task from being deleted. Ownership has been
565 // transferred to the task queue when PostDelayedTask was called.
566 return false;
567 }
568 volatile int activity_;
569
570 rtc::SequencedTaskChecker task_checker_;
571 rtc::WeakPtr<VideoSendStreamImpl> send_stream_;
572 bool timed_out_;
573 };
574
575 class VideoSendStreamImpl::EncoderReconfiguredTask : public rtc::QueuedTask {
576 public:
EncoderReconfiguredTask(const rtc::WeakPtr<VideoSendStreamImpl> & send_stream,std::vector<VideoStream> streams,int min_transmit_bitrate_bps)577 EncoderReconfiguredTask(const rtc::WeakPtr<VideoSendStreamImpl>& send_stream,
578 std::vector<VideoStream> streams,
579 int min_transmit_bitrate_bps)
580 : send_stream_(std::move(send_stream)),
581 streams_(std::move(streams)),
582 min_transmit_bitrate_bps_(min_transmit_bitrate_bps) {}
583
584 private:
Run()585 bool Run() override {
586 if (send_stream_)
587 send_stream_->OnEncoderConfigurationChanged(std::move(streams_),
588 min_transmit_bitrate_bps_);
589 return true;
590 }
591
592 rtc::WeakPtr<VideoSendStreamImpl> send_stream_;
593 std::vector<VideoStream> streams_;
594 int min_transmit_bitrate_bps_;
595 };
596
VideoSendStream(int num_cpu_cores,ProcessThread * module_process_thread,rtc::TaskQueue * worker_queue,CallStats * call_stats,CongestionController * congestion_controller,PacketRouter * packet_router,BitrateAllocator * bitrate_allocator,SendDelayStats * send_delay_stats,VieRemb * remb,RtcEventLog * event_log,VideoSendStream::Config config,VideoEncoderConfig encoder_config,const std::map<uint32_t,RtpState> & suspended_ssrcs)597 VideoSendStream::VideoSendStream(
598 int num_cpu_cores,
599 ProcessThread* module_process_thread,
600 rtc::TaskQueue* worker_queue,
601 CallStats* call_stats,
602 CongestionController* congestion_controller,
603 PacketRouter* packet_router,
604 BitrateAllocator* bitrate_allocator,
605 SendDelayStats* send_delay_stats,
606 VieRemb* remb,
607 RtcEventLog* event_log,
608 VideoSendStream::Config config,
609 VideoEncoderConfig encoder_config,
610 const std::map<uint32_t, RtpState>& suspended_ssrcs)
611 : worker_queue_(worker_queue),
612 thread_sync_event_(false /* manual_reset */, false),
613 stats_proxy_(Clock::GetRealTimeClock(),
614 config,
615 encoder_config.content_type),
616 config_(std::move(config)) {
617 vie_encoder_.reset(new ViEEncoder(
618 num_cpu_cores, &stats_proxy_, config_.encoder_settings,
619 config_.pre_encode_callback, config_.post_encode_callback));
620 worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(new ConstructionTask(
621 &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(),
622 module_process_thread, call_stats, congestion_controller, packet_router,
623 bitrate_allocator, send_delay_stats, remb, event_log, &config_,
624 encoder_config.max_bitrate_bps, suspended_ssrcs)));
625
626 // Wait for ConstructionTask to complete so that |send_stream_| can be used.
627 // |module_process_thread| must be registered and deregistered on the thread
628 // it was created on.
629 thread_sync_event_.Wait(rtc::Event::kForever);
630 send_stream_->RegisterProcessThread(module_process_thread);
631 // TODO(sprang): Enable this also for regular video calls if it works well.
632 if (encoder_config.content_type == VideoEncoderConfig::ContentType::kScreen) {
633 // Only signal target bitrate for screenshare streams, for now.
634 vie_encoder_->SetBitrateObserver(send_stream_.get());
635 }
636 vie_encoder_->RegisterProcessThread(module_process_thread);
637
638 ReconfigureVideoEncoder(std::move(encoder_config));
639 }
640
~VideoSendStream()641 VideoSendStream::~VideoSendStream() {
642 RTC_DCHECK_RUN_ON(&thread_checker_);
643 RTC_DCHECK(!send_stream_);
644 }
645
LoadStateObserver()646 CPULoadStateObserver* VideoSendStream::LoadStateObserver() {
647 return vie_encoder_.get();
648 }
649
Start()650 void VideoSendStream::Start() {
651 RTC_DCHECK_RUN_ON(&thread_checker_);
652 LOG(LS_INFO) << "VideoSendStream::Start";
653 VideoSendStreamImpl* send_stream = send_stream_.get();
654 worker_queue_->PostTask([this, send_stream] {
655 send_stream->Start();
656 thread_sync_event_.Set();
657 });
658
659 // It is expected that after VideoSendStream::Start has been called, incoming
660 // frames are not dropped in ViEEncoder. To ensure this, Start has to be
661 // synchronized.
662 thread_sync_event_.Wait(rtc::Event::kForever);
663 }
664
Stop()665 void VideoSendStream::Stop() {
666 RTC_DCHECK_RUN_ON(&thread_checker_);
667 LOG(LS_INFO) << "VideoSendStream::Stop";
668 VideoSendStreamImpl* send_stream = send_stream_.get();
669 worker_queue_->PostTask([send_stream] { send_stream->Stop(); });
670 }
671
SetSource(rtc::VideoSourceInterface<webrtc::VideoFrame> * source,const DegradationPreference & degradation_preference)672 void VideoSendStream::SetSource(
673 rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
674 const DegradationPreference& degradation_preference) {
675 RTC_DCHECK_RUN_ON(&thread_checker_);
676 vie_encoder_->SetSource(source, degradation_preference);
677 }
678
ReconfigureVideoEncoder(VideoEncoderConfig config)679 void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
680 // TODO(perkj): Some test cases in VideoSendStreamTest call
681 // ReconfigureVideoEncoder from the network thread.
682 // RTC_DCHECK_RUN_ON(&thread_checker_);
683 vie_encoder_->ConfigureEncoder(std::move(config), config_.rtp.max_packet_size,
684 config_.rtp.nack.rtp_history_ms > 0);
685 }
686
GetStats()687 VideoSendStream::Stats VideoSendStream::GetStats() {
688 // TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from
689 // a network thread. See comment in Call::GetStats().
690 // RTC_DCHECK_RUN_ON(&thread_checker_);
691 return stats_proxy_.GetStats();
692 }
693
SignalNetworkState(NetworkState state)694 void VideoSendStream::SignalNetworkState(NetworkState state) {
695 RTC_DCHECK_RUN_ON(&thread_checker_);
696 VideoSendStreamImpl* send_stream = send_stream_.get();
697 worker_queue_->PostTask(
698 [send_stream, state] { send_stream->SignalNetworkState(state); });
699 }
700
StopPermanentlyAndGetRtpStates()701 VideoSendStream::RtpStateMap VideoSendStream::StopPermanentlyAndGetRtpStates() {
702 RTC_DCHECK_RUN_ON(&thread_checker_);
703 vie_encoder_->Stop();
704 vie_encoder_->DeRegisterProcessThread();
705 VideoSendStream::RtpStateMap state_map;
706 send_stream_->DeRegisterProcessThread();
707 worker_queue_->PostTask(
708 std::unique_ptr<rtc::QueuedTask>(new DestructAndGetRtpStateTask(
709 &state_map, std::move(send_stream_), &thread_sync_event_)));
710 thread_sync_event_.Wait(rtc::Event::kForever);
711 return state_map;
712 }
713
SetTransportOverhead(size_t transport_overhead_per_packet)714 void VideoSendStream::SetTransportOverhead(
715 size_t transport_overhead_per_packet) {
716 RTC_DCHECK_RUN_ON(&thread_checker_);
717 VideoSendStreamImpl* send_stream = send_stream_.get();
718 worker_queue_->PostTask([send_stream, transport_overhead_per_packet] {
719 send_stream->SetTransportOverhead(transport_overhead_per_packet);
720 });
721 }
722
DeliverRtcp(const uint8_t * packet,size_t length)723 bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
724 // Called on a network thread.
725 return send_stream_->DeliverRtcp(packet, length);
726 }
727
EnableEncodedFrameRecording(const std::vector<rtc::PlatformFile> & files,size_t byte_limit)728 void VideoSendStream::EnableEncodedFrameRecording(
729 const std::vector<rtc::PlatformFile>& files,
730 size_t byte_limit) {
731 send_stream_->EnableEncodedFrameRecording(files, byte_limit);
732 }
733
VideoSendStreamImpl(SendStatisticsProxy * stats_proxy,rtc::TaskQueue * worker_queue,CallStats * call_stats,CongestionController * congestion_controller,PacketRouter * packet_router,BitrateAllocator * bitrate_allocator,SendDelayStats * send_delay_stats,VieRemb * remb,ViEEncoder * vie_encoder,RtcEventLog * event_log,const VideoSendStream::Config * config,int initial_encoder_max_bitrate,std::map<uint32_t,RtpState> suspended_ssrcs)734 VideoSendStreamImpl::VideoSendStreamImpl(
735 SendStatisticsProxy* stats_proxy,
736 rtc::TaskQueue* worker_queue,
737 CallStats* call_stats,
738 CongestionController* congestion_controller,
739 PacketRouter* packet_router,
740 BitrateAllocator* bitrate_allocator,
741 SendDelayStats* send_delay_stats,
742 VieRemb* remb,
743 ViEEncoder* vie_encoder,
744 RtcEventLog* event_log,
745 const VideoSendStream::Config* config,
746 int initial_encoder_max_bitrate,
747 std::map<uint32_t, RtpState> suspended_ssrcs)
748 : stats_proxy_(stats_proxy),
749 config_(config),
750 suspended_ssrcs_(std::move(suspended_ssrcs)),
751 module_process_thread_(nullptr),
752 worker_queue_(worker_queue),
753 check_encoder_activity_task_(nullptr),
754 call_stats_(call_stats),
755 congestion_controller_(congestion_controller),
756 packet_router_(packet_router),
757 bitrate_allocator_(bitrate_allocator),
758 remb_(remb),
759 flexfec_sender_(MaybeCreateFlexfecSender(*config_)),
760 max_padding_bitrate_(0),
761 encoder_min_bitrate_bps_(0),
762 encoder_max_bitrate_bps_(initial_encoder_max_bitrate),
763 encoder_target_rate_bps_(0),
764 vie_encoder_(vie_encoder),
765 encoder_feedback_(Clock::GetRealTimeClock(),
766 config_->rtp.ssrcs,
767 vie_encoder),
768 protection_bitrate_calculator_(Clock::GetRealTimeClock(), this),
769 bandwidth_observer_(congestion_controller_->GetBitrateController()
770 ->CreateRtcpBandwidthObserver()),
771 rtp_rtcp_modules_(CreateRtpRtcpModules(
772 config_->send_transport,
773 &encoder_feedback_,
774 bandwidth_observer_.get(),
775 congestion_controller_->GetTransportFeedbackObserver(),
776 call_stats_->rtcp_rtt_stats(),
777 congestion_controller_->pacer(),
778 packet_router_,
779 flexfec_sender_.get(),
780 stats_proxy_,
781 send_delay_stats,
782 event_log,
783 congestion_controller_->GetRetransmissionRateLimiter(),
784 this,
785 config_->rtp.ssrcs.size())),
786 payload_router_(rtp_rtcp_modules_,
787 config_->encoder_settings.payload_type),
788 weak_ptr_factory_(this),
789 overhead_bytes_per_packet_(0),
790 transport_overhead_bytes_per_packet_(0) {
791 RTC_DCHECK_RUN_ON(worker_queue_);
792 LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString();
793 weak_ptr_ = weak_ptr_factory_.GetWeakPtr();
794 module_process_thread_checker_.DetachFromThread();
795
796 RTC_DCHECK(!config_->rtp.ssrcs.empty());
797 RTC_DCHECK(call_stats_);
798 RTC_DCHECK(congestion_controller_);
799 RTC_DCHECK(remb_);
800
801 congestion_controller_->EnablePeriodicAlrProbing(
802 config_->periodic_alr_bandwidth_probing);
803
804 // RTP/RTCP initialization.
805 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
806 packet_router_->AddRtpModule(rtp_rtcp);
807 }
808
809 for (size_t i = 0; i < config_->rtp.extensions.size(); ++i) {
810 const std::string& extension = config_->rtp.extensions[i].uri;
811 int id = config_->rtp.extensions[i].id;
812 // One-byte-extension local identifiers are in the range 1-14 inclusive.
813 RTC_DCHECK_GE(id, 1);
814 RTC_DCHECK_LE(id, 14);
815 RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
816 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
817 RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension(
818 StringToRtpExtensionType(extension), id));
819 }
820 }
821
822 remb_->AddRembSender(rtp_rtcp_modules_[0]);
823 rtp_rtcp_modules_[0]->SetREMBStatus(true);
824
825 ConfigureProtection();
826 ConfigureSsrcs();
827
828 // Configure the mid for each of the rtp modules
829 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
830 rtp_rtcp->SetMID(config_->rtp.mid.c_str());
831 }
832
833 // TODO(pbos): Should we set CNAME on all RTP modules?
834 rtp_rtcp_modules_.front()->SetCNAME(config_->rtp.c_name.c_str());
835
836 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
837 rtp_rtcp->RegisterRtcpStatisticsCallback(stats_proxy_);
838 rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(stats_proxy_);
839 rtp_rtcp->SetMaxRtpPacketSize(config_->rtp.max_packet_size);
840 rtp_rtcp->RegisterVideoSendPayload(
841 config_->encoder_settings.payload_type,
842 config_->encoder_settings.payload_name.c_str());
843 }
844
845 RTC_DCHECK(config_->encoder_settings.encoder);
846 RTC_DCHECK_GE(config_->encoder_settings.payload_type, 0);
847 RTC_DCHECK_LE(config_->encoder_settings.payload_type, 127);
848
849 vie_encoder_->SetStartBitrate(bitrate_allocator_->GetStartBitrate(this));
850
851 // Only request rotation at the source when we positively know that the remote
852 // side doesn't support the rotation extension. This allows us to prepare the
853 // encoder in the expectation that rotation is supported - which is the common
854 // case.
855 bool rotation_applied =
856 std::find_if(config_->rtp.extensions.begin(),
857 config_->rtp.extensions.end(),
858 [](const RtpExtension& extension) {
859 return extension.uri == RtpExtension::kVideoRotationUri;
860 }) == config_->rtp.extensions.end();
861
862 vie_encoder_->SetSink(this, rotation_applied);
863 }
864
RegisterProcessThread(ProcessThread * module_process_thread)865 void VideoSendStreamImpl::RegisterProcessThread(
866 ProcessThread* module_process_thread) {
867 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
868 RTC_DCHECK(!module_process_thread_);
869 module_process_thread_ = module_process_thread;
870
871 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
872 module_process_thread_->RegisterModule(rtp_rtcp);
873 }
874
DeRegisterProcessThread()875 void VideoSendStreamImpl::DeRegisterProcessThread() {
876 RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
877 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
878 module_process_thread_->DeRegisterModule(rtp_rtcp);
879 }
880
~VideoSendStreamImpl()881 VideoSendStreamImpl::~VideoSendStreamImpl() {
882 RTC_DCHECK_RUN_ON(worker_queue_);
883 RTC_DCHECK(!payload_router_.IsActive())
884 << "VideoSendStreamImpl::Stop not called";
885 LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString();
886
887 rtp_rtcp_modules_[0]->SetREMBStatus(false);
888 remb_->RemoveRembSender(rtp_rtcp_modules_[0]);
889
890 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
891 packet_router_->RemoveRtpModule(rtp_rtcp);
892 delete rtp_rtcp;
893 }
894 }
895
DeliverRtcp(const uint8_t * packet,size_t length)896 bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) {
897 // Runs on a network thread.
898 RTC_DCHECK(!worker_queue_->IsCurrent());
899 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
900 rtp_rtcp->IncomingRtcpPacket(packet, length);
901 return true;
902 }
903
Start()904 void VideoSendStreamImpl::Start() {
905 RTC_DCHECK_RUN_ON(worker_queue_);
906 LOG(LS_INFO) << "VideoSendStream::Start";
907 if (payload_router_.IsActive())
908 return;
909 TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start");
910 payload_router_.SetActive(true);
911
912 bitrate_allocator_->AddObserver(
913 this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
914 max_padding_bitrate_, !config_->suspend_below_min_bitrate);
915
916 // Start monitoring encoder activity.
917 {
918 rtc::CritScope lock(&encoder_activity_crit_sect_);
919 RTC_DCHECK(!check_encoder_activity_task_);
920 check_encoder_activity_task_ = new CheckEncoderActivityTask(weak_ptr_);
921 worker_queue_->PostDelayedTask(
922 std::unique_ptr<rtc::QueuedTask>(check_encoder_activity_task_),
923 CheckEncoderActivityTask::kEncoderTimeOutMs);
924 }
925
926 vie_encoder_->SendKeyFrame();
927 }
928
Stop()929 void VideoSendStreamImpl::Stop() {
930 RTC_DCHECK_RUN_ON(worker_queue_);
931 LOG(LS_INFO) << "VideoSendStream::Stop";
932 if (!payload_router_.IsActive())
933 return;
934 TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop");
935 payload_router_.SetActive(false);
936 bitrate_allocator_->RemoveObserver(this);
937 {
938 rtc::CritScope lock(&encoder_activity_crit_sect_);
939 check_encoder_activity_task_->Stop();
940 check_encoder_activity_task_ = nullptr;
941 }
942 vie_encoder_->OnBitrateUpdated(0, 0, 0);
943 stats_proxy_->OnSetEncoderTargetRate(0);
944 }
945
SignalEncoderTimedOut()946 void VideoSendStreamImpl::SignalEncoderTimedOut() {
947 RTC_DCHECK_RUN_ON(worker_queue_);
948 // If the encoder has not produced anything the last kEncoderTimeOutMs and it
949 // is supposed to, deregister as BitrateAllocatorObserver. This can happen
950 // if a camera stops producing frames.
951 if (encoder_target_rate_bps_ > 0) {
952 LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out.";
953 bitrate_allocator_->RemoveObserver(this);
954 }
955 }
956
OnBitrateAllocationUpdated(const BitrateAllocation & allocation)957 void VideoSendStreamImpl::OnBitrateAllocationUpdated(
958 const BitrateAllocation& allocation) {
959 payload_router_.OnBitrateAllocationUpdated(allocation);
960 }
961
SignalEncoderActive()962 void VideoSendStreamImpl::SignalEncoderActive() {
963 RTC_DCHECK_RUN_ON(worker_queue_);
964 LOG(LS_INFO) << "SignalEncoderActive, Encoder is active.";
965 bitrate_allocator_->AddObserver(
966 this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
967 max_padding_bitrate_, !config_->suspend_below_min_bitrate);
968 }
969
OnEncoderConfigurationChanged(std::vector<VideoStream> streams,int min_transmit_bitrate_bps)970 void VideoSendStreamImpl::OnEncoderConfigurationChanged(
971 std::vector<VideoStream> streams,
972 int min_transmit_bitrate_bps) {
973 if (!worker_queue_->IsCurrent()) {
974 worker_queue_->PostTask(
975 std::unique_ptr<rtc::QueuedTask>(new EncoderReconfiguredTask(
976 weak_ptr_, std::move(streams), min_transmit_bitrate_bps)));
977 return;
978 }
979 RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
980 TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged");
981 RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
982 RTC_DCHECK_RUN_ON(worker_queue_);
983
984 const int kEncoderMinBitrateBps = 30000;
985 encoder_min_bitrate_bps_ =
986 std::max(streams[0].min_bitrate_bps, kEncoderMinBitrateBps);
987 encoder_max_bitrate_bps_ = 0;
988 for (const auto& stream : streams)
989 encoder_max_bitrate_bps_ += stream.max_bitrate_bps;
990 max_padding_bitrate_ = CalculateMaxPadBitrateBps(
991 streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate);
992
993 // Clear stats for disabled layers.
994 for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
995 stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]);
996 }
997
998 size_t number_of_temporal_layers =
999 streams.back().temporal_layer_thresholds_bps.size() + 1;
1000 protection_bitrate_calculator_.SetEncodingData(
1001 streams[0].width, streams[0].height, number_of_temporal_layers,
1002 config_->rtp.max_packet_size);
1003
1004 if (payload_router_.IsActive()) {
1005 // The send stream is started already. Update the allocator with new bitrate
1006 // limits.
1007 bitrate_allocator_->AddObserver(
1008 this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
1009 max_padding_bitrate_, !config_->suspend_below_min_bitrate);
1010 }
1011 }
1012
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info,const RTPFragmentationHeader * fragmentation)1013 EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
1014 const EncodedImage& encoded_image,
1015 const CodecSpecificInfo* codec_specific_info,
1016 const RTPFragmentationHeader* fragmentation) {
1017 // Encoded is called on whatever thread the real encoder implementation run
1018 // on. In the case of hardware encoders, there might be several encoders
1019 // running in parallel on different threads.
1020 if (config_->post_encode_callback) {
1021 config_->post_encode_callback->EncodedFrameCallback(
1022 EncodedFrame(encoded_image._buffer, encoded_image._length,
1023 encoded_image._frameType));
1024 }
1025 {
1026 rtc::CritScope lock(&encoder_activity_crit_sect_);
1027 if (check_encoder_activity_task_)
1028 check_encoder_activity_task_->UpdateEncoderActivity();
1029 }
1030
1031 protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image);
1032 EncodedImageCallback::Result result = payload_router_.OnEncodedImage(
1033 encoded_image, codec_specific_info, fragmentation);
1034
1035 RTC_DCHECK(codec_specific_info);
1036
1037 int layer = codec_specific_info->codecType == kVideoCodecVP8
1038 ? codec_specific_info->codecSpecific.VP8.simulcastIdx
1039 : 0;
1040 {
1041 rtc::CritScope lock(&ivf_writers_crit_);
1042 if (file_writers_[layer].get()) {
1043 bool ok = file_writers_[layer]->WriteFrame(
1044 encoded_image, codec_specific_info->codecType);
1045 RTC_DCHECK(ok);
1046 }
1047 }
1048
1049 return result;
1050 }
1051
ConfigureProtection()1052 void VideoSendStreamImpl::ConfigureProtection() {
1053 RTC_DCHECK_RUN_ON(worker_queue_);
1054
1055 // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender.
1056 const bool flexfec_enabled = (flexfec_sender_ != nullptr);
1057
1058 // Consistency of NACK and RED+ULPFEC parameters is checked in this function.
1059 const bool nack_enabled = config_->rtp.nack.rtp_history_ms > 0;
1060 int red_payload_type = config_->rtp.ulpfec.red_payload_type;
1061 int ulpfec_payload_type = config_->rtp.ulpfec.ulpfec_payload_type;
1062
1063 // Shorthands.
1064 auto IsRedEnabled = [&]() { return red_payload_type >= 0; };
1065 auto DisableRed = [&]() { red_payload_type = -1; };
1066 auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; };
1067 auto DisableUlpfec = [&]() { ulpfec_payload_type = -1; };
1068
1069 // If enabled, FlexFEC takes priority over RED+ULPFEC.
1070 if (flexfec_enabled) {
1071 // We can safely disable RED here, because if the remote supports FlexFEC,
1072 // we know that it has a receiver without the RED/RTX workaround.
1073 // See http://crbug.com/webrtc/6650 for more information.
1074 if (IsRedEnabled()) {
1075 LOG(LS_INFO) << "Both FlexFEC and RED are configured. Disabling RED.";
1076 DisableRed();
1077 }
1078 if (IsUlpfecEnabled()) {
1079 LOG(LS_INFO)
1080 << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC.";
1081 DisableUlpfec();
1082 }
1083 }
1084
1085 // Payload types without picture ID cannot determine that a stream is complete
1086 // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance)
1087 // is a waste of bandwidth since FEC packets still have to be transmitted.
1088 // Note that this is not the case with FlexFEC.
1089 if (nack_enabled && IsUlpfecEnabled() &&
1090 !PayloadTypeSupportsSkippingFecPackets(
1091 config_->encoder_settings.payload_name)) {
1092 LOG(LS_WARNING)
1093 << "Transmitting payload type without picture ID using "
1094 "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets "
1095 "also have to be retransmitted. Disabling ULPFEC.";
1096 DisableUlpfec();
1097 }
1098
1099 // Verify payload types.
1100 //
1101 // Due to how old receivers work, we need to always send RED if it has been
1102 // negotiated. This is a remnant of an old RED/RTX workaround, see
1103 // https://codereview.webrtc.org/2469093003.
1104 // TODO(brandtr): This change went into M56, so we can remove it in ~M59.
1105 // At that time, we can disable RED whenever ULPFEC is disabled, as there is
1106 // no point in using RED without ULPFEC.
1107 if (IsRedEnabled()) {
1108 RTC_DCHECK_GE(red_payload_type, 0);
1109 RTC_DCHECK_LE(red_payload_type, 127);
1110 }
1111 if (IsUlpfecEnabled()) {
1112 RTC_DCHECK_GE(ulpfec_payload_type, 0);
1113 RTC_DCHECK_LE(ulpfec_payload_type, 127);
1114 if (!IsRedEnabled()) {
1115 LOG(LS_WARNING)
1116 << "ULPFEC is enabled but RED is disabled. Disabling ULPFEC.";
1117 DisableUlpfec();
1118 }
1119 }
1120
1121 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1122 // Set NACK.
1123 rtp_rtcp->SetStorePacketsStatus(
1124 true,
1125 kMinSendSidePacketHistorySize);
1126 // Set RED/ULPFEC information.
1127 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1128 rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type);
1129 }
1130 }
1131
1132 // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
1133 // so enable that logic if either of those FEC schemes are enabled.
1134 protection_bitrate_calculator_.SetProtectionMethod(
1135 flexfec_enabled || IsUlpfecEnabled(), nack_enabled);
1136 }
1137
ConfigureSsrcs()1138 void VideoSendStreamImpl::ConfigureSsrcs() {
1139 RTC_DCHECK_RUN_ON(worker_queue_);
1140 // Configure regular SSRCs.
1141 bool has_rids = false;
1142 if (config_->rtp.rids.size() != 0) {
1143 has_rids = true;
1144 // if we have rids, we must have a rid entry for every ssrc (even if it's "")
1145 RTC_DCHECK(config_->rtp.rids.size() == config_->rtp.ssrcs.size());
1146 }
1147 for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) {
1148 uint32_t ssrc = config_->rtp.ssrcs[i];
1149 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
1150 rtp_rtcp->SetSSRC(ssrc);
1151 if (has_rids && config_->rtp.rids[i] != "") {
1152 rtp_rtcp->SetRID(config_->rtp.rids[i].c_str());
1153 }
1154
1155 // Restore RTP state if previous existed.
1156 VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
1157 if (it != suspended_ssrcs_.end())
1158 rtp_rtcp->SetRtpState(it->second);
1159 }
1160
1161 // Set up RTX if available.
1162 if (config_->rtp.rtx.ssrcs.empty())
1163 return;
1164
1165 // Configure RTX SSRCs.
1166 RTC_DCHECK_EQ(config_->rtp.rtx.ssrcs.size(), config_->rtp.ssrcs.size());
1167 for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) {
1168 uint32_t ssrc = config_->rtp.rtx.ssrcs[i];
1169 RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
1170 rtp_rtcp->SetRtxSsrc(ssrc);
1171 VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
1172 if (it != suspended_ssrcs_.end())
1173 rtp_rtcp->SetRtxState(it->second);
1174 }
1175
1176 // Configure RTX payload types.
1177 RTC_DCHECK_GE(config_->rtp.rtx.payload_type, 0);
1178 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1179 rtp_rtcp->SetRtxSendPayloadType(config_->rtp.rtx.payload_type,
1180 config_->encoder_settings.payload_type);
1181 rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads);
1182 }
1183 if (config_->rtp.ulpfec.red_payload_type != -1 &&
1184 config_->rtp.ulpfec.red_rtx_payload_type != -1) {
1185 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1186 rtp_rtcp->SetRtxSendPayloadType(config_->rtp.ulpfec.red_rtx_payload_type,
1187 config_->rtp.ulpfec.red_payload_type);
1188 }
1189 }
1190 }
1191
GetRtpStates() const1192 std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
1193 RTC_DCHECK_RUN_ON(worker_queue_);
1194 std::map<uint32_t, RtpState> rtp_states;
1195 for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) {
1196 uint32_t ssrc = config_->rtp.ssrcs[i];
1197 RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC());
1198 rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtpState();
1199 }
1200
1201 for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) {
1202 uint32_t ssrc = config_->rtp.rtx.ssrcs[i];
1203 rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState();
1204 }
1205
1206 return rtp_states;
1207 }
1208
SignalNetworkState(NetworkState state)1209 void VideoSendStreamImpl::SignalNetworkState(NetworkState state) {
1210 RTC_DCHECK_RUN_ON(worker_queue_);
1211 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1212 rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_->rtp.rtcp_mode
1213 : RtcpMode::kOff);
1214 }
1215 }
1216
OnBitrateUpdated(uint32_t bitrate_bps,uint8_t fraction_loss,int64_t rtt,int64_t probing_interval_ms)1217 uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps,
1218 uint8_t fraction_loss,
1219 int64_t rtt,
1220 int64_t probing_interval_ms) {
1221 RTC_DCHECK_RUN_ON(worker_queue_);
1222 RTC_DCHECK(payload_router_.IsActive())
1223 << "VideoSendStream::Start has not been called.";
1224
1225 if (webrtc::field_trial::FindFullName("WebRTC-SendSideBwe-WithOverhead") ==
1226 "Enabled") {
1227 // Subtract total overhead (transport + rtp) from bitrate.
1228 size_t rtp_overhead;
1229 {
1230 rtc::CritScope lock(&overhead_bytes_per_packet_crit_);
1231 rtp_overhead = overhead_bytes_per_packet_;
1232 }
1233 RTC_CHECK_GE(rtp_overhead, 0);
1234 RTC_DCHECK_LT(rtp_overhead, config_->rtp.max_packet_size);
1235 if (rtp_overhead >= config_->rtp.max_packet_size) {
1236 LOG(LS_WARNING) << "RTP overhead (" << rtp_overhead << " bytes)"
1237 << "exceeds maximum packet size ("
1238 << config_->rtp.max_packet_size << " bytes)";
1239
1240 bitrate_bps = 0;
1241 } else {
1242 bitrate_bps =
1243 static_cast<uint32_t>(static_cast<uint64_t>(bitrate_bps) *
1244 (config_->rtp.max_packet_size - rtp_overhead) /
1245 (config_->rtp.max_packet_size +
1246 transport_overhead_bytes_per_packet_));
1247 }
1248 }
1249
1250 // Get the encoder target rate. It is the estimated network rate -
1251 // protection overhead.
1252 encoder_target_rate_bps_ = protection_bitrate_calculator_.SetTargetRates(
1253 bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, rtt);
1254 uint32_t protection_bitrate = bitrate_bps - encoder_target_rate_bps_;
1255
1256 encoder_target_rate_bps_ =
1257 std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_);
1258 vie_encoder_->OnBitrateUpdated(encoder_target_rate_bps_, fraction_loss, rtt);
1259 stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_);
1260 return protection_bitrate;
1261 }
1262
EnableEncodedFrameRecording(const std::vector<rtc::PlatformFile> & files,size_t byte_limit)1263 void VideoSendStreamImpl::EnableEncodedFrameRecording(
1264 const std::vector<rtc::PlatformFile>& files,
1265 size_t byte_limit) {
1266 {
1267 rtc::CritScope lock(&ivf_writers_crit_);
1268 for (unsigned int i = 0; i < kMaxSimulcastStreams; ++i) {
1269 if (i < files.size()) {
1270 file_writers_[i] = IvfFileWriter::Wrap(rtc::File(files[i]), byte_limit);
1271 } else {
1272 file_writers_[i].reset();
1273 }
1274 }
1275 }
1276
1277 if (!files.empty()) {
1278 // Make a keyframe appear as early as possible in the logs, to give actually
1279 // decodable output.
1280 vie_encoder_->SendKeyFrame();
1281 }
1282 }
1283
ProtectionRequest(const FecProtectionParams * delta_params,const FecProtectionParams * key_params,uint32_t * sent_video_rate_bps,uint32_t * sent_nack_rate_bps,uint32_t * sent_fec_rate_bps)1284 int VideoSendStreamImpl::ProtectionRequest(
1285 const FecProtectionParams* delta_params,
1286 const FecProtectionParams* key_params,
1287 uint32_t* sent_video_rate_bps,
1288 uint32_t* sent_nack_rate_bps,
1289 uint32_t* sent_fec_rate_bps) {
1290 RTC_DCHECK_RUN_ON(worker_queue_);
1291 *sent_video_rate_bps = 0;
1292 *sent_nack_rate_bps = 0;
1293 *sent_fec_rate_bps = 0;
1294 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1295 uint32_t not_used = 0;
1296 uint32_t module_video_rate = 0;
1297 uint32_t module_fec_rate = 0;
1298 uint32_t module_nack_rate = 0;
1299 rtp_rtcp->SetFecParameters(*delta_params, *key_params);
1300 rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate,
1301 &module_nack_rate);
1302 *sent_video_rate_bps += module_video_rate;
1303 *sent_nack_rate_bps += module_nack_rate;
1304 *sent_fec_rate_bps += module_fec_rate;
1305 }
1306 return 0;
1307 }
1308
OnOverheadChanged(size_t overhead_bytes_per_packet)1309 void VideoSendStreamImpl::OnOverheadChanged(size_t overhead_bytes_per_packet) {
1310 rtc::CritScope lock(&overhead_bytes_per_packet_crit_);
1311 overhead_bytes_per_packet_ = overhead_bytes_per_packet;
1312 }
1313
SetTransportOverhead(size_t transport_overhead_bytes_per_packet)1314 void VideoSendStreamImpl::SetTransportOverhead(
1315 size_t transport_overhead_bytes_per_packet) {
1316 if (transport_overhead_bytes_per_packet >= static_cast<int>(kPathMTU)) {
1317 LOG(LS_ERROR) << "Transport overhead exceeds size of ethernet frame";
1318 return;
1319 }
1320
1321 transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet;
1322
1323 congestion_controller_->SetTransportOverhead(
1324 transport_overhead_bytes_per_packet_);
1325
1326 size_t rtp_packet_size =
1327 std::min(config_->rtp.max_packet_size,
1328 kPathMTU - transport_overhead_bytes_per_packet_);
1329
1330 for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1331 rtp_rtcp->SetMaxRtpPacketSize(rtp_packet_size);
1332 }
1333 }
1334
1335 } // namespace internal
1336 } // namespace webrtc
1337