1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_receive_stream2.h"
12
13 #include <stdlib.h>
14 #include <string.h>
15
16 #include <algorithm>
17 #include <memory>
18 #include <set>
19 #include <string>
20 #include <utility>
21
22 #include "absl/algorithm/container.h"
23 #include "absl/types/optional.h"
24 #include "api/array_view.h"
25 #include "api/crypto/frame_decryptor_interface.h"
26 #include "api/video/encoded_image.h"
27 #include "api/video_codecs/sdp_video_format.h"
28 #include "api/video_codecs/video_codec.h"
29 #include "api/video_codecs/video_decoder_factory.h"
30 #include "api/video_codecs/video_encoder.h"
31 #include "call/rtp_stream_receiver_controller_interface.h"
32 #include "call/rtx_receive_stream.h"
33 #include "common_video/include/incoming_video_stream.h"
34 #include "media/base/h264_profile_level_id.h"
35 #include "modules/video_coding/include/video_codec_interface.h"
36 #include "modules/video_coding/include/video_coding_defines.h"
37 #include "modules/video_coding/include/video_error_codes.h"
38 #include "modules/video_coding/timing.h"
39 #include "modules/video_coding/utility/vp8_header_parser.h"
40 #include "rtc_base/checks.h"
41 #include "rtc_base/experiments/keyframe_interval_settings.h"
42 #include "rtc_base/location.h"
43 #include "rtc_base/logging.h"
44 #include "rtc_base/strings/string_builder.h"
45 #include "rtc_base/system/thread_registry.h"
46 #include "rtc_base/time_utils.h"
47 #include "rtc_base/trace_event.h"
48 #include "system_wrappers/include/clock.h"
49 #include "system_wrappers/include/field_trial.h"
50 #include "video/call_stats2.h"
51 #include "video/frame_dumping_decoder.h"
52 #include "video/receive_statistics_proxy2.h"
53
54 namespace webrtc {
55
56 namespace internal {
57 constexpr int VideoReceiveStream2::kMaxWaitForKeyFrameMs;
58
59 namespace {
60
61 using video_coding::EncodedFrame;
62 using ReturnReason = video_coding::FrameBuffer::ReturnReason;
63
64 constexpr int kMinBaseMinimumDelayMs = 0;
65 constexpr int kMaxBaseMinimumDelayMs = 10000;
66
67 constexpr int kMaxWaitForFrameMs = 3000;
68
69 // Concrete instance of RecordableEncodedFrame wrapping needed content
70 // from video_coding::EncodedFrame.
71 class WebRtcRecordableEncodedFrame : public RecordableEncodedFrame {
72 public:
WebRtcRecordableEncodedFrame(const EncodedFrame & frame)73 explicit WebRtcRecordableEncodedFrame(const EncodedFrame& frame)
74 : buffer_(frame.GetEncodedData()),
75 render_time_ms_(frame.RenderTime()),
76 codec_(frame.CodecSpecific()->codecType),
77 is_key_frame_(frame.FrameType() == VideoFrameType::kVideoFrameKey),
78 resolution_{frame.EncodedImage()._encodedWidth,
79 frame.EncodedImage()._encodedHeight} {
80 if (frame.ColorSpace()) {
81 color_space_ = *frame.ColorSpace();
82 }
83 }
84
85 // VideoEncodedSinkInterface::FrameBuffer
encoded_buffer() const86 rtc::scoped_refptr<const EncodedImageBufferInterface> encoded_buffer()
87 const override {
88 return buffer_;
89 }
90
color_space() const91 absl::optional<webrtc::ColorSpace> color_space() const override {
92 return color_space_;
93 }
94
codec() const95 VideoCodecType codec() const override { return codec_; }
96
is_key_frame() const97 bool is_key_frame() const override { return is_key_frame_; }
98
resolution() const99 EncodedResolution resolution() const override { return resolution_; }
100
render_time() const101 Timestamp render_time() const override {
102 return Timestamp::Millis(render_time_ms_);
103 }
104
105 private:
106 rtc::scoped_refptr<EncodedImageBufferInterface> buffer_;
107 int64_t render_time_ms_;
108 VideoCodecType codec_;
109 bool is_key_frame_;
110 EncodedResolution resolution_;
111 absl::optional<webrtc::ColorSpace> color_space_;
112 };
113
CreateDecoderVideoCodec(const VideoReceiveStream::Decoder & decoder)114 VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
115 VideoCodec codec;
116 codec.codecType = PayloadStringToCodecType(decoder.video_format.name);
117
118 if (codec.codecType == kVideoCodecVP8) {
119 *(codec.VP8()) = VideoEncoder::GetDefaultVp8Settings();
120 } else if (codec.codecType == kVideoCodecVP9) {
121 *(codec.VP9()) = VideoEncoder::GetDefaultVp9Settings();
122 } else if (codec.codecType == kVideoCodecH264) {
123 *(codec.H264()) = VideoEncoder::GetDefaultH264Settings();
124 } else if (codec.codecType == kVideoCodecMultiplex) {
125 VideoReceiveStream::Decoder associated_decoder = decoder;
126 associated_decoder.video_format =
127 SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP9));
128 VideoCodec associated_codec = CreateDecoderVideoCodec(associated_decoder);
129 associated_codec.codecType = kVideoCodecMultiplex;
130 return associated_codec;
131 }
132
133 FieldTrialOptional<int> width("w");
134 FieldTrialOptional<int> height("h");
135 ParseFieldTrial(
136 {&width, &height},
137 field_trial::FindFullName("WebRTC-Video-InitialDecoderResolution"));
138 if (width && height) {
139 codec.width = width.Value();
140 codec.height = height.Value();
141 } else {
142 codec.width = 320;
143 codec.height = 180;
144 }
145
146 const int kDefaultStartBitrate = 300;
147 codec.startBitrate = codec.minBitrate = codec.maxBitrate =
148 kDefaultStartBitrate;
149
150 return codec;
151 }
152
153 // Video decoder class to be used for unknown codecs. Doesn't support decoding
154 // but logs messages to LS_ERROR.
155 class NullVideoDecoder : public webrtc::VideoDecoder {
156 public:
InitDecode(const webrtc::VideoCodec * codec_settings,int32_t number_of_cores)157 int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
158 int32_t number_of_cores) override {
159 RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder.";
160 return WEBRTC_VIDEO_CODEC_OK;
161 }
162
Decode(const webrtc::EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)163 int32_t Decode(const webrtc::EncodedImage& input_image,
164 bool missing_frames,
165 int64_t render_time_ms) override {
166 RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding.";
167 return WEBRTC_VIDEO_CODEC_OK;
168 }
169
RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback)170 int32_t RegisterDecodeCompleteCallback(
171 webrtc::DecodedImageCallback* callback) override {
172 RTC_LOG(LS_ERROR)
173 << "Can't register decode complete callback on NullVideoDecoder.";
174 return WEBRTC_VIDEO_CODEC_OK;
175 }
176
Release()177 int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; }
178
ImplementationName() const179 const char* ImplementationName() const override { return "NullVideoDecoder"; }
180 };
181
182 // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround.
183 // Maximum time between frames before resetting the FrameBuffer to avoid RTP
184 // timestamps wraparound to affect FrameBuffer.
185 constexpr int kInactiveStreamThresholdMs = 600000; // 10 minutes.
186
187 } // namespace
188
VideoReceiveStream2(TaskQueueFactory * task_queue_factory,TaskQueueBase * current_queue,RtpStreamReceiverControllerInterface * receiver_controller,int num_cpu_cores,PacketRouter * packet_router,VideoReceiveStream::Config config,ProcessThread * process_thread,CallStats * call_stats,Clock * clock,VCMTiming * timing)189 VideoReceiveStream2::VideoReceiveStream2(
190 TaskQueueFactory* task_queue_factory,
191 TaskQueueBase* current_queue,
192 RtpStreamReceiverControllerInterface* receiver_controller,
193 int num_cpu_cores,
194 PacketRouter* packet_router,
195 VideoReceiveStream::Config config,
196 ProcessThread* process_thread,
197 CallStats* call_stats,
198 Clock* clock,
199 VCMTiming* timing)
200 : task_queue_factory_(task_queue_factory),
201 transport_adapter_(config.rtcp_send_transport),
202 config_(std::move(config)),
203 num_cpu_cores_(num_cpu_cores),
204 worker_thread_(current_queue),
205 clock_(clock),
206 call_stats_(call_stats),
207 source_tracker_(clock_),
208 stats_proxy_(&config_, clock_, worker_thread_),
209 rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
210 timing_(timing),
211 video_receiver_(clock_, timing_.get()),
212 rtp_video_stream_receiver_(worker_thread_,
213 clock_,
214 &transport_adapter_,
215 call_stats->AsRtcpRttStats(),
216 packet_router,
217 &config_,
218 rtp_receive_statistics_.get(),
219 &stats_proxy_,
220 &stats_proxy_,
221 process_thread,
222 this, // NackSender
223 nullptr, // Use default KeyFrameRequestSender
224 this, // OnCompleteFrameCallback
225 config_.frame_decryptor,
226 config_.frame_transformer),
227 rtp_stream_sync_(current_queue, this),
228 max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
229 .MaxWaitForKeyframeMs()
230 .value_or(kMaxWaitForKeyFrameMs)),
231 max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
232 .MaxWaitForFrameMs()
233 .value_or(kMaxWaitForFrameMs)),
234 low_latency_renderer_enabled_("enabled", true),
235 low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
236 true),
237 decode_queue_(task_queue_factory_->CreateTaskQueue(
238 "DecodingQueue",
239 TaskQueueFactory::Priority::HIGH)) {
240 RTC_LOG(LS_INFO) << "VideoReceiveStream2: " << config_.ToString();
241
242 RTC_DCHECK(worker_thread_);
243 RTC_DCHECK(config_.renderer);
244 RTC_DCHECK(call_stats_);
245
246 module_process_sequence_checker_.Detach();
247
248 RTC_DCHECK(!config_.decoders.empty());
249 RTC_CHECK(config_.decoder_factory);
250 std::set<int> decoder_payload_types;
251 for (const Decoder& decoder : config_.decoders) {
252 RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
253 decoder_payload_types.end())
254 << "Duplicate payload type (" << decoder.payload_type
255 << ") for different decoders.";
256 decoder_payload_types.insert(decoder.payload_type);
257 }
258
259 timing_->set_render_delay(config_.render_delay_ms);
260
261 frame_buffer_.reset(
262 new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_));
263
264 // Register with RtpStreamReceiverController.
265 media_receiver_ = receiver_controller->CreateReceiver(
266 config_.rtp.remote_ssrc, &rtp_video_stream_receiver_);
267 if (config_.rtp.rtx_ssrc) {
268 rtx_receive_stream_ = std::make_unique<RtxReceiveStream>(
269 &rtp_video_stream_receiver_, config.rtp.rtx_associated_payload_types,
270 config_.rtp.remote_ssrc, rtp_receive_statistics_.get());
271 rtx_receiver_ = receiver_controller->CreateReceiver(
272 config_.rtp.rtx_ssrc, rtx_receive_stream_.get());
273 } else {
274 rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc,
275 true);
276 }
277
278 ParseFieldTrial({&low_latency_renderer_enabled_,
279 &low_latency_renderer_include_predecode_buffer_},
280 field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
281 }
282
~VideoReceiveStream2()283 VideoReceiveStream2::~VideoReceiveStream2() {
284 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
285 RTC_LOG(LS_INFO) << "~VideoReceiveStream2: " << config_.ToString();
286 Stop();
287 }
288
SignalNetworkState(NetworkState state)289 void VideoReceiveStream2::SignalNetworkState(NetworkState state) {
290 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
291 rtp_video_stream_receiver_.SignalNetworkState(state);
292 }
293
DeliverRtcp(const uint8_t * packet,size_t length)294 bool VideoReceiveStream2::DeliverRtcp(const uint8_t* packet, size_t length) {
295 return rtp_video_stream_receiver_.DeliverRtcp(packet, length);
296 }
297
SetSync(Syncable * audio_syncable)298 void VideoReceiveStream2::SetSync(Syncable* audio_syncable) {
299 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
300 rtp_stream_sync_.ConfigureSync(audio_syncable);
301 }
302
Start()303 void VideoReceiveStream2::Start() {
304 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
305
306 if (decoder_running_) {
307 return;
308 }
309
310 const bool protected_by_fec = config_.rtp.protected_by_flexfec ||
311 rtp_video_stream_receiver_.IsUlpfecEnabled();
312
313 if (rtp_video_stream_receiver_.IsRetransmissionsEnabled() &&
314 protected_by_fec) {
315 frame_buffer_->SetProtectionMode(kProtectionNackFEC);
316 }
317
318 transport_adapter_.Enable();
319 rtc::VideoSinkInterface<VideoFrame>* renderer = nullptr;
320 if (config_.enable_prerenderer_smoothing) {
321 incoming_video_stream_.reset(new IncomingVideoStream(
322 task_queue_factory_, config_.render_delay_ms, this));
323 renderer = incoming_video_stream_.get();
324 } else {
325 renderer = this;
326 }
327
328 for (const Decoder& decoder : config_.decoders) {
329 std::unique_ptr<VideoDecoder> video_decoder =
330 config_.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format,
331 config_.stream_id);
332 // If we still have no valid decoder, we have to create a "Null" decoder
333 // that ignores all calls. The reason we can get into this state is that the
334 // old decoder factory interface doesn't have a way to query supported
335 // codecs.
336 if (!video_decoder) {
337 video_decoder = std::make_unique<NullVideoDecoder>();
338 }
339
340 std::string decoded_output_file =
341 field_trial::FindFullName("WebRTC-DecoderDataDumpDirectory");
342 // Because '/' can't be used inside a field trial parameter, we use ';'
343 // instead.
344 // This is only relevant to WebRTC-DecoderDataDumpDirectory
345 // field trial. ';' is chosen arbitrary. Even though it's a legal character
346 // in some file systems, we can sacrifice ability to use it in the path to
347 // dumped video, since it's developers-only feature for debugging.
348 absl::c_replace(decoded_output_file, ';', '/');
349 if (!decoded_output_file.empty()) {
350 char filename_buffer[256];
351 rtc::SimpleStringBuilder ssb(filename_buffer);
352 ssb << decoded_output_file << "/webrtc_receive_stream_"
353 << this->config_.rtp.remote_ssrc << "-" << rtc::TimeMicros()
354 << ".ivf";
355 video_decoder = CreateFrameDumpingDecoderWrapper(
356 std::move(video_decoder), FileWrapper::OpenWriteOnly(ssb.str()));
357 }
358
359 video_decoders_.push_back(std::move(video_decoder));
360
361 video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
362 decoder.payload_type);
363 VideoCodec codec = CreateDecoderVideoCodec(decoder);
364
365 const bool raw_payload =
366 config_.rtp.raw_payload_types.count(decoder.payload_type) > 0;
367 rtp_video_stream_receiver_.AddReceiveCodec(decoder.payload_type, codec,
368 decoder.video_format.parameters,
369 raw_payload);
370 RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec(
371 decoder.payload_type, &codec, num_cpu_cores_));
372 }
373
374 RTC_DCHECK(renderer != nullptr);
375 video_stream_decoder_.reset(
376 new VideoStreamDecoder(&video_receiver_, &stats_proxy_, renderer));
377
378 // Make sure we register as a stats observer *after* we've prepared the
379 // |video_stream_decoder_|.
380 call_stats_->RegisterStatsObserver(this);
381
382 // Start decoding on task queue.
383 video_receiver_.DecoderThreadStarting();
384 stats_proxy_.DecoderThreadStarting();
385 decode_queue_.PostTask([this] {
386 RTC_DCHECK_RUN_ON(&decode_queue_);
387 decoder_stopped_ = false;
388 StartNextDecode();
389 });
390 decoder_running_ = true;
391 rtp_video_stream_receiver_.StartReceive();
392 }
393
Stop()394 void VideoReceiveStream2::Stop() {
395 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
396 rtp_video_stream_receiver_.StopReceive();
397
398 stats_proxy_.OnUniqueFramesCounted(
399 rtp_video_stream_receiver_.GetUniqueFramesSeen());
400
401 decode_queue_.PostTask([this] { frame_buffer_->Stop(); });
402
403 call_stats_->DeregisterStatsObserver(this);
404
405 if (decoder_running_) {
406 rtc::Event done;
407 decode_queue_.PostTask([this, &done] {
408 RTC_DCHECK_RUN_ON(&decode_queue_);
409 decoder_stopped_ = true;
410 done.Set();
411 });
412 done.Wait(rtc::Event::kForever);
413
414 decoder_running_ = false;
415 video_receiver_.DecoderThreadStopped();
416 stats_proxy_.DecoderThreadStopped();
417 // Deregister external decoders so they are no longer running during
418 // destruction. This effectively stops the VCM since the decoder thread is
419 // stopped, the VCM is deregistered and no asynchronous decoder threads are
420 // running.
421 for (const Decoder& decoder : config_.decoders)
422 video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type);
423
424 UpdateHistograms();
425 }
426
427 video_stream_decoder_.reset();
428 incoming_video_stream_.reset();
429 transport_adapter_.Disable();
430 }
431
GetStats() const432 VideoReceiveStream::Stats VideoReceiveStream2::GetStats() const {
433 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
434 VideoReceiveStream2::Stats stats = stats_proxy_.GetStats();
435 stats.total_bitrate_bps = 0;
436 StreamStatistician* statistician =
437 rtp_receive_statistics_->GetStatistician(stats.ssrc);
438 if (statistician) {
439 stats.rtp_stats = statistician->GetStats();
440 stats.total_bitrate_bps = statistician->BitrateReceived();
441 }
442 if (config_.rtp.rtx_ssrc) {
443 StreamStatistician* rtx_statistician =
444 rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc);
445 if (rtx_statistician)
446 stats.total_bitrate_bps += rtx_statistician->BitrateReceived();
447 }
448 return stats;
449 }
450
UpdateHistograms()451 void VideoReceiveStream2::UpdateHistograms() {
452 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
453 absl::optional<int> fraction_lost;
454 StreamDataCounters rtp_stats;
455 StreamStatistician* statistician =
456 rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc);
457 if (statistician) {
458 fraction_lost = statistician->GetFractionLostInPercent();
459 rtp_stats = statistician->GetReceiveStreamDataCounters();
460 }
461 if (config_.rtp.rtx_ssrc) {
462 StreamStatistician* rtx_statistician =
463 rtp_receive_statistics_->GetStatistician(config_.rtp.rtx_ssrc);
464 if (rtx_statistician) {
465 StreamDataCounters rtx_stats =
466 rtx_statistician->GetReceiveStreamDataCounters();
467 stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, &rtx_stats);
468 return;
469 }
470 }
471 stats_proxy_.UpdateHistograms(fraction_lost, rtp_stats, nullptr);
472 }
473
AddSecondarySink(RtpPacketSinkInterface * sink)474 void VideoReceiveStream2::AddSecondarySink(RtpPacketSinkInterface* sink) {
475 rtp_video_stream_receiver_.AddSecondarySink(sink);
476 }
477
RemoveSecondarySink(const RtpPacketSinkInterface * sink)478 void VideoReceiveStream2::RemoveSecondarySink(
479 const RtpPacketSinkInterface* sink) {
480 rtp_video_stream_receiver_.RemoveSecondarySink(sink);
481 }
482
SetBaseMinimumPlayoutDelayMs(int delay_ms)483 bool VideoReceiveStream2::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
484 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
485 if (delay_ms < kMinBaseMinimumDelayMs || delay_ms > kMaxBaseMinimumDelayMs) {
486 return false;
487 }
488
489 base_minimum_playout_delay_ms_ = delay_ms;
490 UpdatePlayoutDelays();
491 return true;
492 }
493
GetBaseMinimumPlayoutDelayMs() const494 int VideoReceiveStream2::GetBaseMinimumPlayoutDelayMs() const {
495 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
496 return base_minimum_playout_delay_ms_;
497 }
498
OnFrame(const VideoFrame & video_frame)499 void VideoReceiveStream2::OnFrame(const VideoFrame& video_frame) {
500 VideoFrameMetaData frame_meta(video_frame, clock_->CurrentTime());
501
502 worker_thread_->PostTask(
503 ToQueuedTask(task_safety_, [frame_meta, this]() {
504 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
505 int64_t video_playout_ntp_ms;
506 int64_t sync_offset_ms;
507 double estimated_freq_khz;
508 if (rtp_stream_sync_.GetStreamSyncOffsetInMs(
509 frame_meta.rtp_timestamp, frame_meta.render_time_ms(),
510 &video_playout_ntp_ms, &sync_offset_ms, &estimated_freq_khz)) {
511 stats_proxy_.OnSyncOffsetUpdated(video_playout_ntp_ms, sync_offset_ms,
512 estimated_freq_khz);
513 }
514 stats_proxy_.OnRenderedFrame(frame_meta);
515 }));
516
517 source_tracker_.OnFrameDelivered(video_frame.packet_infos());
518 config_.renderer->OnFrame(video_frame);
519 }
520
SetFrameDecryptor(rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor)521 void VideoReceiveStream2::SetFrameDecryptor(
522 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
523 rtp_video_stream_receiver_.SetFrameDecryptor(std::move(frame_decryptor));
524 }
525
SetDepacketizerToDecoderFrameTransformer(rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)526 void VideoReceiveStream2::SetDepacketizerToDecoderFrameTransformer(
527 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
528 rtp_video_stream_receiver_.SetDepacketizerToDecoderFrameTransformer(
529 std::move(frame_transformer));
530 }
531
SendNack(const std::vector<uint16_t> & sequence_numbers,bool buffering_allowed)532 void VideoReceiveStream2::SendNack(
533 const std::vector<uint16_t>& sequence_numbers,
534 bool buffering_allowed) {
535 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
536 RTC_DCHECK(buffering_allowed);
537 rtp_video_stream_receiver_.RequestPacketRetransmit(sequence_numbers);
538 }
539
RequestKeyFrame(int64_t timestamp_ms)540 void VideoReceiveStream2::RequestKeyFrame(int64_t timestamp_ms) {
541 // Running on worker_sequence_checker_.
542 // Called from RtpVideoStreamReceiver (rtp_video_stream_receiver_ is
543 // ultimately responsible).
544 rtp_video_stream_receiver_.RequestKeyFrame();
545 decode_queue_.PostTask([this, timestamp_ms]() {
546 RTC_DCHECK_RUN_ON(&decode_queue_);
547 last_keyframe_request_ms_ = timestamp_ms;
548 });
549 }
550
OnCompleteFrame(std::unique_ptr<video_coding::EncodedFrame> frame)551 void VideoReceiveStream2::OnCompleteFrame(
552 std::unique_ptr<video_coding::EncodedFrame> frame) {
553 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
554
555 // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround.
556 int64_t time_now_ms = clock_->TimeInMilliseconds();
557 if (last_complete_frame_time_ms_ > 0 &&
558 time_now_ms - last_complete_frame_time_ms_ > kInactiveStreamThresholdMs) {
559 frame_buffer_->Clear();
560 }
561 last_complete_frame_time_ms_ = time_now_ms;
562
563 const VideoPlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_;
564 if (playout_delay.min_ms >= 0) {
565 frame_minimum_playout_delay_ms_ = playout_delay.min_ms;
566 UpdatePlayoutDelays();
567 }
568
569 if (playout_delay.max_ms >= 0) {
570 frame_maximum_playout_delay_ms_ = playout_delay.max_ms;
571 UpdatePlayoutDelays();
572 }
573
574 int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame));
575 if (last_continuous_pid != -1)
576 rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid);
577 }
578
OnRttUpdate(int64_t avg_rtt_ms,int64_t max_rtt_ms)579 void VideoReceiveStream2::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) {
580 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
581 frame_buffer_->UpdateRtt(max_rtt_ms);
582 rtp_video_stream_receiver_.UpdateRtt(max_rtt_ms);
583 stats_proxy_.OnRttUpdate(avg_rtt_ms);
584 }
585
id() const586 uint32_t VideoReceiveStream2::id() const {
587 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
588 return config_.rtp.remote_ssrc;
589 }
590
GetInfo() const591 absl::optional<Syncable::Info> VideoReceiveStream2::GetInfo() const {
592 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
593 absl::optional<Syncable::Info> info =
594 rtp_video_stream_receiver_.GetSyncInfo();
595
596 if (!info)
597 return absl::nullopt;
598
599 info->current_delay_ms = timing_->TargetVideoDelay();
600 return info;
601 }
602
GetPlayoutRtpTimestamp(uint32_t * rtp_timestamp,int64_t * time_ms) const603 bool VideoReceiveStream2::GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp,
604 int64_t* time_ms) const {
605 RTC_NOTREACHED();
606 return 0;
607 }
608
SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms,int64_t time_ms)609 void VideoReceiveStream2::SetEstimatedPlayoutNtpTimestampMs(
610 int64_t ntp_timestamp_ms,
611 int64_t time_ms) {
612 RTC_NOTREACHED();
613 }
614
SetMinimumPlayoutDelay(int delay_ms)615 bool VideoReceiveStream2::SetMinimumPlayoutDelay(int delay_ms) {
616 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
617 syncable_minimum_playout_delay_ms_ = delay_ms;
618 UpdatePlayoutDelays();
619 return true;
620 }
621
GetMaxWaitMs() const622 int64_t VideoReceiveStream2::GetMaxWaitMs() const {
623 return keyframe_required_ ? max_wait_for_keyframe_ms_
624 : max_wait_for_frame_ms_;
625 }
626
StartNextDecode()627 void VideoReceiveStream2::StartNextDecode() {
628 // Running on the decode thread.
629 TRACE_EVENT0("webrtc", "VideoReceiveStream2::StartNextDecode");
630 frame_buffer_->NextFrame(
631 GetMaxWaitMs(), keyframe_required_, &decode_queue_,
632 /* encoded frame handler */
633 [this](std::unique_ptr<EncodedFrame> frame, ReturnReason res) {
634 RTC_DCHECK_EQ(frame == nullptr, res == ReturnReason::kTimeout);
635 RTC_DCHECK_EQ(frame != nullptr, res == ReturnReason::kFrameFound);
636 decode_queue_.PostTask([this, frame = std::move(frame)]() mutable {
637 RTC_DCHECK_RUN_ON(&decode_queue_);
638 if (decoder_stopped_)
639 return;
640 if (frame) {
641 HandleEncodedFrame(std::move(frame));
642 } else {
643 int64_t now_ms = clock_->TimeInMilliseconds();
644 worker_thread_->PostTask(ToQueuedTask(
645 task_safety_, [this, now_ms, wait_ms = GetMaxWaitMs()]() {
646 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
647 HandleFrameBufferTimeout(now_ms, wait_ms);
648 }));
649 }
650 StartNextDecode();
651 });
652 });
653 }
654
HandleEncodedFrame(std::unique_ptr<EncodedFrame> frame)655 void VideoReceiveStream2::HandleEncodedFrame(
656 std::unique_ptr<EncodedFrame> frame) {
657 // Running on |decode_queue_|.
658 int64_t now_ms = clock_->TimeInMilliseconds();
659
660 // Current OnPreDecode only cares about QP for VP8.
661 int qp = -1;
662 if (frame->CodecSpecific()->codecType == kVideoCodecVP8) {
663 if (!vp8::GetQp(frame->data(), frame->size(), &qp)) {
664 RTC_LOG(LS_WARNING) << "Failed to extract QP from VP8 video frame";
665 }
666 }
667 stats_proxy_.OnPreDecode(frame->CodecSpecific()->codecType, qp);
668
669 bool force_request_key_frame = false;
670 int64_t decoded_frame_picture_id = -1;
671
672 const bool keyframe_request_is_due =
673 now_ms >= (last_keyframe_request_ms_ + max_wait_for_keyframe_ms_);
674
675 int decode_result = video_receiver_.Decode(frame.get());
676 if (decode_result == WEBRTC_VIDEO_CODEC_OK ||
677 decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) {
678 keyframe_required_ = false;
679 frame_decoded_ = true;
680
681 decoded_frame_picture_id = frame->id.picture_id;
682
683 if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME)
684 force_request_key_frame = true;
685 } else if (!frame_decoded_ || !keyframe_required_ ||
686 keyframe_request_is_due) {
687 keyframe_required_ = true;
688 // TODO(philipel): Remove this keyframe request when downstream project
689 // has been fixed.
690 force_request_key_frame = true;
691 }
692
693 bool received_frame_is_keyframe =
694 frame->FrameType() == VideoFrameType::kVideoFrameKey;
695
696 worker_thread_->PostTask(ToQueuedTask(
697 task_safety_,
698 [this, now_ms, received_frame_is_keyframe, force_request_key_frame,
699 decoded_frame_picture_id, keyframe_request_is_due]() {
700 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
701
702 if (decoded_frame_picture_id != -1)
703 rtp_video_stream_receiver_.FrameDecoded(decoded_frame_picture_id);
704
705 HandleKeyFrameGeneration(received_frame_is_keyframe, now_ms,
706 force_request_key_frame,
707 keyframe_request_is_due);
708 }));
709
710 if (encoded_frame_buffer_function_) {
711 frame->Retain();
712 encoded_frame_buffer_function_(WebRtcRecordableEncodedFrame(*frame));
713 }
714 }
715
HandleKeyFrameGeneration(bool received_frame_is_keyframe,int64_t now_ms,bool always_request_key_frame,bool keyframe_request_is_due)716 void VideoReceiveStream2::HandleKeyFrameGeneration(
717 bool received_frame_is_keyframe,
718 int64_t now_ms,
719 bool always_request_key_frame,
720 bool keyframe_request_is_due) {
721 // Running on worker_sequence_checker_.
722
723 bool request_key_frame = always_request_key_frame;
724
725 // Repeat sending keyframe requests if we've requested a keyframe.
726 if (keyframe_generation_requested_) {
727 if (received_frame_is_keyframe) {
728 keyframe_generation_requested_ = false;
729 } else if (keyframe_request_is_due) {
730 if (!IsReceivingKeyFrame(now_ms)) {
731 request_key_frame = true;
732 }
733 } else {
734 // It hasn't been long enough since the last keyframe request, do nothing.
735 }
736 }
737
738 if (request_key_frame) {
739 // HandleKeyFrameGeneration is initated from the decode thread -
740 // RequestKeyFrame() triggers a call back to the decode thread.
741 // Perhaps there's a way to avoid that.
742 RequestKeyFrame(now_ms);
743 }
744 }
745
HandleFrameBufferTimeout(int64_t now_ms,int64_t wait_ms)746 void VideoReceiveStream2::HandleFrameBufferTimeout(int64_t now_ms,
747 int64_t wait_ms) {
748 // Running on |worker_sequence_checker_|.
749 absl::optional<int64_t> last_packet_ms =
750 rtp_video_stream_receiver_.LastReceivedPacketMs();
751
752 // To avoid spamming keyframe requests for a stream that is not active we
753 // check if we have received a packet within the last 5 seconds.
754 const bool stream_is_active =
755 last_packet_ms && now_ms - *last_packet_ms < 5000;
756 if (!stream_is_active)
757 stats_proxy_.OnStreamInactive();
758
759 if (stream_is_active && !IsReceivingKeyFrame(now_ms) &&
760 (!config_.crypto_options.sframe.require_frame_encryption ||
761 rtp_video_stream_receiver_.IsDecryptable())) {
762 RTC_LOG(LS_WARNING) << "No decodable frame in " << wait_ms
763 << " ms, requesting keyframe.";
764 RequestKeyFrame(now_ms);
765 }
766 }
767
IsReceivingKeyFrame(int64_t timestamp_ms) const768 bool VideoReceiveStream2::IsReceivingKeyFrame(int64_t timestamp_ms) const {
769 // Running on worker_sequence_checker_.
770 absl::optional<int64_t> last_keyframe_packet_ms =
771 rtp_video_stream_receiver_.LastReceivedKeyframePacketMs();
772
773 // If we recently have been receiving packets belonging to a keyframe then
774 // we assume a keyframe is currently being received.
775 bool receiving_keyframe =
776 last_keyframe_packet_ms &&
777 timestamp_ms - *last_keyframe_packet_ms < max_wait_for_keyframe_ms_;
778 return receiving_keyframe;
779 }
780
UpdatePlayoutDelays() const781 void VideoReceiveStream2::UpdatePlayoutDelays() const {
782 // Running on worker_sequence_checker_.
783 const int minimum_delay_ms =
784 std::max({frame_minimum_playout_delay_ms_, base_minimum_playout_delay_ms_,
785 syncable_minimum_playout_delay_ms_});
786 if (minimum_delay_ms >= 0) {
787 timing_->set_min_playout_delay(minimum_delay_ms);
788 if (frame_minimum_playout_delay_ms_ == 0 &&
789 frame_maximum_playout_delay_ms_ > 0 && low_latency_renderer_enabled_) {
790 // TODO(kron): Estimate frame rate from video stream.
791 constexpr double kFrameRate = 60.0;
792 // Convert playout delay in ms to number of frames.
793 int max_composition_delay_in_frames = std::lrint(
794 static_cast<double>(frame_maximum_playout_delay_ms_ * kFrameRate) /
795 rtc::kNumMillisecsPerSec);
796 if (low_latency_renderer_include_predecode_buffer_) {
797 // Subtract frames in buffer.
798 max_composition_delay_in_frames = std::max<int16_t>(
799 max_composition_delay_in_frames - frame_buffer_->Size(), 0);
800 }
801 timing_->SetMaxCompositionDelayInFrames(
802 absl::make_optional(max_composition_delay_in_frames));
803 }
804 }
805
806 const int maximum_delay_ms = frame_maximum_playout_delay_ms_;
807 if (maximum_delay_ms >= 0) {
808 timing_->set_max_playout_delay(maximum_delay_ms);
809 }
810 }
811
GetSources() const812 std::vector<webrtc::RtpSource> VideoReceiveStream2::GetSources() const {
813 return source_tracker_.GetSources();
814 }
815
816 VideoReceiveStream2::RecordingState
SetAndGetRecordingState(RecordingState state,bool generate_key_frame)817 VideoReceiveStream2::SetAndGetRecordingState(RecordingState state,
818 bool generate_key_frame) {
819 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
820 rtc::Event event;
821
822 // Save old state, set the new state.
823 RecordingState old_state;
824
825 decode_queue_.PostTask(
826 [this, &event, &old_state, callback = std::move(state.callback),
827 generate_key_frame,
828 last_keyframe_request = state.last_keyframe_request_ms.value_or(0)] {
829 RTC_DCHECK_RUN_ON(&decode_queue_);
830 old_state.callback = std::move(encoded_frame_buffer_function_);
831 encoded_frame_buffer_function_ = std::move(callback);
832
833 old_state.last_keyframe_request_ms = last_keyframe_request_ms_;
834 last_keyframe_request_ms_ = generate_key_frame
835 ? clock_->TimeInMilliseconds()
836 : last_keyframe_request;
837
838 event.Set();
839 });
840
841 old_state.keyframe_needed = keyframe_generation_requested_;
842
843 if (generate_key_frame) {
844 rtp_video_stream_receiver_.RequestKeyFrame();
845 keyframe_generation_requested_ = true;
846 } else {
847 keyframe_generation_requested_ = state.keyframe_needed;
848 }
849
850 event.Wait(rtc::Event::kForever);
851 return old_state;
852 }
853
GenerateKeyFrame()854 void VideoReceiveStream2::GenerateKeyFrame() {
855 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
856 RequestKeyFrame(clock_->TimeInMilliseconds());
857 keyframe_generation_requested_ = true;
858 }
859
860 } // namespace internal
861 } // namespace webrtc
862