1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "call/rtp_video_sender.h"
12
13 #include <atomic>
14 #include <memory>
15 #include <string>
16
17 #include "call/rtp_transport_controller_send.h"
18 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
19 #include "modules/rtp_rtcp/source/byte_io.h"
20 #include "modules/rtp_rtcp/source/rtcp_packet/nack.h"
21 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
22 #include "modules/rtp_rtcp/source/rtp_packet.h"
23 #include "modules/video_coding/fec_controller_default.h"
24 #include "modules/video_coding/include/video_codec_interface.h"
25 #include "rtc_base/rate_limiter.h"
26 #include "test/field_trial.h"
27 #include "test/gmock.h"
28 #include "test/gtest.h"
29 #include "test/mock_frame_transformer.h"
30 #include "test/mock_transport.h"
31 #include "test/scenario/scenario.h"
32 #include "test/time_controller/simulated_time_controller.h"
33 #include "video/call_stats.h"
34 #include "video/send_delay_stats.h"
35 #include "video/send_statistics_proxy.h"
36
37 using ::testing::_;
38 using ::testing::NiceMock;
39 using ::testing::SaveArg;
40 using ::testing::SizeIs;
41
42 namespace webrtc {
43 namespace {
44 const int8_t kPayloadType = 96;
45 const uint32_t kSsrc1 = 12345;
46 const uint32_t kSsrc2 = 23456;
47 const uint32_t kRtxSsrc1 = 34567;
48 const uint32_t kRtxSsrc2 = 45678;
49 const int16_t kInitialPictureId1 = 222;
50 const int16_t kInitialPictureId2 = 44;
51 const int16_t kInitialTl0PicIdx1 = 99;
52 const int16_t kInitialTl0PicIdx2 = 199;
53 const int64_t kRetransmitWindowSizeMs = 500;
54 const int kTransportsSequenceExtensionId = 7;
55 const int kDependencyDescriptorExtensionId = 8;
56
57 class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver {
58 public:
59 MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override));
60 };
61
CreateObservers(RtcpRttStats * rtcp_rtt_stats,RtcpIntraFrameObserver * intra_frame_callback,RtcpStatisticsCallback * rtcp_stats,ReportBlockDataObserver * report_block_data_observer,StreamDataCountersCallback * rtp_stats,BitrateStatisticsObserver * bitrate_observer,FrameCountObserver * frame_count_observer,RtcpPacketTypeCounterObserver * rtcp_type_observer,SendSideDelayObserver * send_delay_observer,SendPacketObserver * send_packet_observer)62 RtpSenderObservers CreateObservers(
63 RtcpRttStats* rtcp_rtt_stats,
64 RtcpIntraFrameObserver* intra_frame_callback,
65 RtcpStatisticsCallback* rtcp_stats,
66 ReportBlockDataObserver* report_block_data_observer,
67 StreamDataCountersCallback* rtp_stats,
68 BitrateStatisticsObserver* bitrate_observer,
69 FrameCountObserver* frame_count_observer,
70 RtcpPacketTypeCounterObserver* rtcp_type_observer,
71 SendSideDelayObserver* send_delay_observer,
72 SendPacketObserver* send_packet_observer) {
73 RtpSenderObservers observers;
74 observers.rtcp_rtt_stats = rtcp_rtt_stats;
75 observers.intra_frame_callback = intra_frame_callback;
76 observers.rtcp_loss_notification_observer = nullptr;
77 observers.rtcp_stats = rtcp_stats;
78 observers.report_block_data_observer = report_block_data_observer;
79 observers.rtp_stats = rtp_stats;
80 observers.bitrate_observer = bitrate_observer;
81 observers.frame_count_observer = frame_count_observer;
82 observers.rtcp_type_observer = rtcp_type_observer;
83 observers.send_delay_observer = send_delay_observer;
84 observers.send_packet_observer = send_packet_observer;
85 return observers;
86 }
87
GetBitrateConfig()88 BitrateConstraints GetBitrateConfig() {
89 BitrateConstraints bitrate_config;
90 bitrate_config.min_bitrate_bps = 30000;
91 bitrate_config.start_bitrate_bps = 300000;
92 bitrate_config.max_bitrate_bps = 3000000;
93 return bitrate_config;
94 }
95
CreateVideoSendStreamConfig(Transport * transport,const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type)96 VideoSendStream::Config CreateVideoSendStreamConfig(
97 Transport* transport,
98 const std::vector<uint32_t>& ssrcs,
99 const std::vector<uint32_t>& rtx_ssrcs,
100 int payload_type) {
101 VideoSendStream::Config config(transport);
102 config.rtp.ssrcs = ssrcs;
103 config.rtp.rtx.ssrcs = rtx_ssrcs;
104 config.rtp.payload_type = payload_type;
105 config.rtp.rtx.payload_type = payload_type + 1;
106 config.rtp.nack.rtp_history_ms = 1000;
107 config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri,
108 kTransportsSequenceExtensionId);
109 config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::kUri,
110 kDependencyDescriptorExtensionId);
111 return config;
112 }
113
114 class RtpVideoSenderTestFixture {
115 public:
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states,FrameCountObserver * frame_count_observer,rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)116 RtpVideoSenderTestFixture(
117 const std::vector<uint32_t>& ssrcs,
118 const std::vector<uint32_t>& rtx_ssrcs,
119 int payload_type,
120 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
121 FrameCountObserver* frame_count_observer,
122 rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
123 : time_controller_(Timestamp::Millis(1000000)),
124 config_(CreateVideoSendStreamConfig(&transport_,
125 ssrcs,
126 rtx_ssrcs,
127 payload_type)),
128 send_delay_stats_(time_controller_.GetClock()),
129 bitrate_config_(GetBitrateConfig()),
130 transport_controller_(
131 time_controller_.GetClock(),
132 &event_log_,
133 nullptr,
134 nullptr,
135 bitrate_config_,
136 time_controller_.CreateProcessThread("PacerThread"),
137 time_controller_.GetTaskQueueFactory(),
138 &field_trials_),
139 process_thread_(time_controller_.CreateProcessThread("test_thread")),
140 call_stats_(time_controller_.GetClock(), process_thread_.get()),
141 stats_proxy_(time_controller_.GetClock(),
142 config_,
143 VideoEncoderConfig::ContentType::kRealtimeVideo),
144 retransmission_rate_limiter_(time_controller_.GetClock(),
145 kRetransmitWindowSizeMs) {
146 transport_controller_.EnsureStarted();
147 std::map<uint32_t, RtpState> suspended_ssrcs;
148 router_ = std::make_unique<RtpVideoSender>(
149 time_controller_.GetClock(), suspended_ssrcs, suspended_payload_states,
150 config_.rtp, config_.rtcp_report_interval_ms, &transport_,
151 CreateObservers(&call_stats_, &encoder_feedback_, &stats_proxy_,
152 &stats_proxy_, &stats_proxy_, &stats_proxy_,
153 frame_count_observer, &stats_proxy_, &stats_proxy_,
154 &send_delay_stats_),
155 &transport_controller_, &event_log_, &retransmission_rate_limiter_,
156 std::make_unique<FecControllerDefault>(time_controller_.GetClock()),
157 nullptr, CryptoOptions{}, frame_transformer);
158 }
159
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states,FrameCountObserver * frame_count_observer)160 RtpVideoSenderTestFixture(
161 const std::vector<uint32_t>& ssrcs,
162 const std::vector<uint32_t>& rtx_ssrcs,
163 int payload_type,
164 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
165 FrameCountObserver* frame_count_observer)
166 : RtpVideoSenderTestFixture(ssrcs,
167 rtx_ssrcs,
168 payload_type,
169 suspended_payload_states,
170 frame_count_observer,
171 /*frame_transformer=*/nullptr) {}
172
RtpVideoSenderTestFixture(const std::vector<uint32_t> & ssrcs,const std::vector<uint32_t> & rtx_ssrcs,int payload_type,const std::map<uint32_t,RtpPayloadState> & suspended_payload_states)173 RtpVideoSenderTestFixture(
174 const std::vector<uint32_t>& ssrcs,
175 const std::vector<uint32_t>& rtx_ssrcs,
176 int payload_type,
177 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states)
178 : RtpVideoSenderTestFixture(ssrcs,
179 rtx_ssrcs,
180 payload_type,
181 suspended_payload_states,
182 /*frame_count_observer=*/nullptr,
183 /*frame_transformer=*/nullptr) {}
184
router()185 RtpVideoSender* router() { return router_.get(); }
transport()186 MockTransport& transport() { return transport_; }
AdvanceTime(TimeDelta delta)187 void AdvanceTime(TimeDelta delta) { time_controller_.AdvanceTime(delta); }
188
189 private:
190 NiceMock<MockTransport> transport_;
191 NiceMock<MockRtcpIntraFrameObserver> encoder_feedback_;
192 GlobalSimulatedTimeController time_controller_;
193 RtcEventLogNull event_log_;
194 VideoSendStream::Config config_;
195 SendDelayStats send_delay_stats_;
196 BitrateConstraints bitrate_config_;
197 const FieldTrialBasedConfig field_trials_;
198 RtpTransportControllerSend transport_controller_;
199 std::unique_ptr<ProcessThread> process_thread_;
200 // TODO(tommi): Use internal::CallStats.
201 CallStats call_stats_;
202 SendStatisticsProxy stats_proxy_;
203 RateLimiter retransmission_rate_limiter_;
204 std::unique_ptr<RtpVideoSender> router_;
205 };
206 } // namespace
207
TEST(RtpVideoSenderTest,SendOnOneModule)208 TEST(RtpVideoSenderTest, SendOnOneModule) {
209 constexpr uint8_t kPayload = 'a';
210 EncodedImage encoded_image;
211 encoded_image.SetTimestamp(1);
212 encoded_image.capture_time_ms_ = 2;
213 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
214 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
215
216 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
217 EXPECT_NE(EncodedImageCallback::Result::OK,
218 test.router()->OnEncodedImage(encoded_image, nullptr).error);
219
220 test.router()->SetActive(true);
221 EXPECT_EQ(EncodedImageCallback::Result::OK,
222 test.router()->OnEncodedImage(encoded_image, nullptr).error);
223
224 test.router()->SetActive(false);
225 EXPECT_NE(EncodedImageCallback::Result::OK,
226 test.router()->OnEncodedImage(encoded_image, nullptr).error);
227
228 test.router()->SetActive(true);
229 EXPECT_EQ(EncodedImageCallback::Result::OK,
230 test.router()->OnEncodedImage(encoded_image, nullptr).error);
231 }
232
TEST(RtpVideoSenderTest,SendSimulcastSetActive)233 TEST(RtpVideoSenderTest, SendSimulcastSetActive) {
234 constexpr uint8_t kPayload = 'a';
235 EncodedImage encoded_image_1;
236 encoded_image_1.SetTimestamp(1);
237 encoded_image_1.capture_time_ms_ = 2;
238 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey;
239 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
240
241 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
242 kPayloadType, {});
243
244 CodecSpecificInfo codec_info;
245 codec_info.codecType = kVideoCodecVP8;
246
247 test.router()->SetActive(true);
248 EXPECT_EQ(EncodedImageCallback::Result::OK,
249 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error);
250
251 EncodedImage encoded_image_2(encoded_image_1);
252 encoded_image_2.SetSpatialIndex(1);
253 EXPECT_EQ(EncodedImageCallback::Result::OK,
254 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error);
255
256 // Inactive.
257 test.router()->SetActive(false);
258 EXPECT_NE(EncodedImageCallback::Result::OK,
259 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error);
260 EXPECT_NE(EncodedImageCallback::Result::OK,
261 test.router()->OnEncodedImage(encoded_image_2, &codec_info).error);
262 }
263
264 // Tests how setting individual rtp modules to active affects the overall
265 // behavior of the payload router. First sets one module to active and checks
266 // that outgoing data can be sent on this module, and checks that no data can
267 // be sent if both modules are inactive.
TEST(RtpVideoSenderTest,SendSimulcastSetActiveModules)268 TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) {
269 constexpr uint8_t kPayload = 'a';
270 EncodedImage encoded_image_1;
271 encoded_image_1.SetTimestamp(1);
272 encoded_image_1.capture_time_ms_ = 2;
273 encoded_image_1._frameType = VideoFrameType::kVideoFrameKey;
274 encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
275
276 EncodedImage encoded_image_2(encoded_image_1);
277 encoded_image_2.SetSpatialIndex(1);
278
279 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
280 kPayloadType, {});
281 CodecSpecificInfo codec_info;
282 codec_info.codecType = kVideoCodecVP8;
283
284 // Only setting one stream to active will still set the payload router to
285 // active and allow sending data on the active stream.
286 std::vector<bool> active_modules({true, false});
287 test.router()->SetActiveModules(active_modules);
288 EXPECT_EQ(EncodedImageCallback::Result::OK,
289 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error);
290
291 // Setting both streams to inactive will turn the payload router to
292 // inactive.
293 active_modules = {false, false};
294 test.router()->SetActiveModules(active_modules);
295 // An incoming encoded image will not ask the module to send outgoing data
296 // because the payload router is inactive.
297 EXPECT_NE(EncodedImageCallback::Result::OK,
298 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error);
299 EXPECT_NE(EncodedImageCallback::Result::OK,
300 test.router()->OnEncodedImage(encoded_image_1, &codec_info).error);
301 }
302
TEST(RtpVideoSenderTest,CreateWithNoPreviousStates)303 TEST(RtpVideoSenderTest, CreateWithNoPreviousStates) {
304 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
305 kPayloadType, {});
306 test.router()->SetActive(true);
307
308 std::map<uint32_t, RtpPayloadState> initial_states =
309 test.router()->GetRtpPayloadStates();
310 EXPECT_EQ(2u, initial_states.size());
311 EXPECT_NE(initial_states.find(kSsrc1), initial_states.end());
312 EXPECT_NE(initial_states.find(kSsrc2), initial_states.end());
313 }
314
TEST(RtpVideoSenderTest,CreateWithPreviousStates)315 TEST(RtpVideoSenderTest, CreateWithPreviousStates) {
316 const int64_t kState1SharedFrameId = 123;
317 const int64_t kState2SharedFrameId = 234;
318 RtpPayloadState state1;
319 state1.picture_id = kInitialPictureId1;
320 state1.tl0_pic_idx = kInitialTl0PicIdx1;
321 state1.shared_frame_id = kState1SharedFrameId;
322 RtpPayloadState state2;
323 state2.picture_id = kInitialPictureId2;
324 state2.tl0_pic_idx = kInitialTl0PicIdx2;
325 state2.shared_frame_id = kState2SharedFrameId;
326 std::map<uint32_t, RtpPayloadState> states = {{kSsrc1, state1},
327 {kSsrc2, state2}};
328
329 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
330 kPayloadType, states);
331 test.router()->SetActive(true);
332
333 std::map<uint32_t, RtpPayloadState> initial_states =
334 test.router()->GetRtpPayloadStates();
335 EXPECT_EQ(2u, initial_states.size());
336 EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id);
337 EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx);
338 EXPECT_EQ(kInitialPictureId2, initial_states[kSsrc2].picture_id);
339 EXPECT_EQ(kInitialTl0PicIdx2, initial_states[kSsrc2].tl0_pic_idx);
340 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc1].shared_frame_id);
341 EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc2].shared_frame_id);
342 }
343
TEST(RtpVideoSenderTest,FrameCountCallbacks)344 TEST(RtpVideoSenderTest, FrameCountCallbacks) {
345 class MockFrameCountObserver : public FrameCountObserver {
346 public:
347 MOCK_METHOD(void,
348 FrameCountUpdated,
349 (const FrameCounts& frame_counts, uint32_t ssrc),
350 (override));
351 } callback;
352
353 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {},
354 &callback);
355
356 constexpr uint8_t kPayload = 'a';
357 EncodedImage encoded_image;
358 encoded_image.SetTimestamp(1);
359 encoded_image.capture_time_ms_ = 2;
360 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
361 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
362
363 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
364
365 // No callbacks when not active.
366 EXPECT_CALL(callback, FrameCountUpdated).Times(0);
367 EXPECT_NE(EncodedImageCallback::Result::OK,
368 test.router()->OnEncodedImage(encoded_image, nullptr).error);
369 ::testing::Mock::VerifyAndClearExpectations(&callback);
370
371 test.router()->SetActive(true);
372
373 FrameCounts frame_counts;
374 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1))
375 .WillOnce(SaveArg<0>(&frame_counts));
376 EXPECT_EQ(EncodedImageCallback::Result::OK,
377 test.router()->OnEncodedImage(encoded_image, nullptr).error);
378
379 EXPECT_EQ(1, frame_counts.key_frames);
380 EXPECT_EQ(0, frame_counts.delta_frames);
381
382 ::testing::Mock::VerifyAndClearExpectations(&callback);
383
384 encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
385 EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1))
386 .WillOnce(SaveArg<0>(&frame_counts));
387 EXPECT_EQ(EncodedImageCallback::Result::OK,
388 test.router()->OnEncodedImage(encoded_image, nullptr).error);
389
390 EXPECT_EQ(1, frame_counts.key_frames);
391 EXPECT_EQ(1, frame_counts.delta_frames);
392 }
393
394 // Integration test verifying that ack of packet via TransportFeedback means
395 // that the packet is removed from RtpPacketHistory and won't be retransmitted
396 // again.
TEST(RtpVideoSenderTest,DoesNotRetrasmitAckedPackets)397 TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) {
398 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
399 kPayloadType, {});
400 test.router()->SetActive(true);
401
402 constexpr uint8_t kPayload = 'a';
403 EncodedImage encoded_image;
404 encoded_image.SetTimestamp(1);
405 encoded_image.capture_time_ms_ = 2;
406 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
407 encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1));
408
409 // Send two tiny images, mapping to two RTP packets. Capture sequence numbers.
410 std::vector<uint16_t> rtp_sequence_numbers;
411 std::vector<uint16_t> transport_sequence_numbers;
412 EXPECT_CALL(test.transport(), SendRtp)
413 .Times(2)
414 .WillRepeatedly([&rtp_sequence_numbers, &transport_sequence_numbers](
415 const uint8_t* packet, size_t length,
416 const PacketOptions& options) {
417 RtpPacket rtp_packet;
418 EXPECT_TRUE(rtp_packet.Parse(packet, length));
419 rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber());
420 transport_sequence_numbers.push_back(options.packet_id);
421 return true;
422 });
423 EXPECT_EQ(EncodedImageCallback::Result::OK,
424 test.router()->OnEncodedImage(encoded_image, nullptr).error);
425 encoded_image.SetTimestamp(2);
426 encoded_image.capture_time_ms_ = 3;
427 EXPECT_EQ(EncodedImageCallback::Result::OK,
428 test.router()->OnEncodedImage(encoded_image, nullptr).error);
429
430 test.AdvanceTime(TimeDelta::Millis(33));
431
432 // Construct a NACK message for requesting retransmission of both packet.
433 rtcp::Nack nack;
434 nack.SetMediaSsrc(kSsrc1);
435 nack.SetPacketIds(rtp_sequence_numbers);
436 rtc::Buffer nack_buffer = nack.Build();
437
438 std::vector<uint16_t> retransmitted_rtp_sequence_numbers;
439 EXPECT_CALL(test.transport(), SendRtp)
440 .Times(2)
441 .WillRepeatedly([&retransmitted_rtp_sequence_numbers](
442 const uint8_t* packet, size_t length,
443 const PacketOptions& options) {
444 RtpPacket rtp_packet;
445 EXPECT_TRUE(rtp_packet.Parse(packet, length));
446 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
447 // Capture the retransmitted sequence number from the RTX header.
448 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
449 retransmitted_rtp_sequence_numbers.push_back(
450 ByteReader<uint16_t>::ReadBigEndian(payload.data()));
451 return true;
452 });
453 test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size());
454 test.AdvanceTime(TimeDelta::Millis(33));
455
456 // Verify that both packets were retransmitted.
457 EXPECT_EQ(retransmitted_rtp_sequence_numbers, rtp_sequence_numbers);
458
459 // Simulate transport feedback indicating fist packet received, next packet
460 // lost (not other way around as that would trigger early retransmit).
461 StreamFeedbackObserver::StreamPacketInfo lost_packet_feedback;
462 lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0];
463 lost_packet_feedback.ssrc = kSsrc1;
464 lost_packet_feedback.received = false;
465
466 StreamFeedbackObserver::StreamPacketInfo received_packet_feedback;
467 received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1];
468 received_packet_feedback.ssrc = kSsrc1;
469 received_packet_feedback.received = true;
470
471 test.router()->OnPacketFeedbackVector(
472 {lost_packet_feedback, received_packet_feedback});
473
474 // Advance time to make sure retransmission would be allowed and try again.
475 // This time the retransmission should not happen for the first packet since
476 // the history has been notified of the ack and removed the packet. The
477 // second packet, included in the feedback but not marked as received, should
478 // still be retransmitted.
479 test.AdvanceTime(TimeDelta::Millis(33));
480 EXPECT_CALL(test.transport(), SendRtp)
481 .WillOnce([&lost_packet_feedback](const uint8_t* packet, size_t length,
482 const PacketOptions& options) {
483 RtpPacket rtp_packet;
484 EXPECT_TRUE(rtp_packet.Parse(packet, length));
485 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
486 // Capture the retransmitted sequence number from the RTX header.
487 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
488 EXPECT_EQ(lost_packet_feedback.rtp_sequence_number,
489 ByteReader<uint16_t>::ReadBigEndian(payload.data()));
490 return true;
491 });
492 test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size());
493 test.AdvanceTime(TimeDelta::Millis(33));
494 }
495
496 // This tests that we utilize transport wide feedback to retransmit lost
497 // packets. This is tested by dropping all ordirary packets from a "lossy"
498 // stream send along with an secondary untouched stream. The transport wide
499 // feedback packets from the secondary stream allows the sending side to
500 // detect and retreansmit the lost packets from the lossy stream.
TEST(RtpVideoSenderTest,RetransmitsOnTransportWideLossInfo)501 TEST(RtpVideoSenderTest, RetransmitsOnTransportWideLossInfo) {
502 int rtx_packets;
503 test::Scenario s(test_info_);
504 test::CallClientConfig call_conf;
505 // Keeping the bitrate fixed to avoid RTX due to probing.
506 call_conf.transport.rates.max_rate = DataRate::KilobitsPerSec(300);
507 call_conf.transport.rates.start_rate = DataRate::KilobitsPerSec(300);
508 test::NetworkSimulationConfig net_conf;
509 net_conf.bandwidth = DataRate::KilobitsPerSec(300);
510 auto send_node = s.CreateSimulationNode(net_conf);
511 auto* callee = s.CreateClient("return", call_conf);
512 auto* route = s.CreateRoutes(s.CreateClient("send", call_conf), {send_node},
513 callee, {s.CreateSimulationNode(net_conf)});
514
515 test::VideoStreamConfig lossy_config;
516 lossy_config.source.framerate = 5;
517 auto* lossy = s.CreateVideoStream(route->forward(), lossy_config);
518 // The secondary stream acts a driver for transport feedback messages,
519 // ensuring that lost packets on the lossy stream are retransmitted.
520 s.CreateVideoStream(route->forward(), test::VideoStreamConfig());
521
522 send_node->router()->SetFilter([&](const EmulatedIpPacket& packet) {
523 RtpPacket rtp;
524 if (rtp.Parse(packet.data)) {
525 // Drops all regular packets for the lossy stream and counts all RTX
526 // packets. Since no packets are let trough, NACKs can't be triggered
527 // by the receiving side.
528 if (lossy->send()->UsingSsrc(rtp.Ssrc())) {
529 return false;
530 } else if (lossy->send()->UsingRtxSsrc(rtp.Ssrc())) {
531 ++rtx_packets;
532 }
533 }
534 return true;
535 });
536
537 // Run for a short duration and reset counters to avoid counting RTX packets
538 // from initial probing.
539 s.RunFor(TimeDelta::Seconds(1));
540 rtx_packets = 0;
541 int decoded_baseline = 0;
542 callee->SendTask([&decoded_baseline, &lossy]() {
543 decoded_baseline = lossy->receive()->GetStats().frames_decoded;
544 });
545 s.RunFor(TimeDelta::Seconds(1));
546 // We expect both that RTX packets were sent and that an appropriate number of
547 // frames were received. This is somewhat redundant but reduces the risk of
548 // false positives in future regressions (e.g. RTX is send due to probing).
549 EXPECT_GE(rtx_packets, 1);
550 int frames_decoded = 0;
551 callee->SendTask([&decoded_baseline, &frames_decoded, &lossy]() {
552 frames_decoded =
553 lossy->receive()->GetStats().frames_decoded - decoded_baseline;
554 });
555 EXPECT_EQ(frames_decoded, 5);
556 }
557
558 // Integration test verifying that retransmissions are sent for packets which
559 // can be detected as lost early, using transport wide feedback.
TEST(RtpVideoSenderTest,EarlyRetransmits)560 TEST(RtpVideoSenderTest, EarlyRetransmits) {
561 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
562 kPayloadType, {});
563 test.router()->SetActive(true);
564
565 const uint8_t kPayload[1] = {'a'};
566 EncodedImage encoded_image;
567 encoded_image.SetTimestamp(1);
568 encoded_image.capture_time_ms_ = 2;
569 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
570 encoded_image.SetEncodedData(
571 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
572 encoded_image.SetSpatialIndex(0);
573
574 CodecSpecificInfo codec_specific;
575 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
576
577 // Send two tiny images, mapping to single RTP packets. Capture sequence
578 // numbers.
579 uint16_t frame1_rtp_sequence_number = 0;
580 uint16_t frame1_transport_sequence_number = 0;
581 EXPECT_CALL(test.transport(), SendRtp)
582 .WillOnce(
583 [&frame1_rtp_sequence_number, &frame1_transport_sequence_number](
584 const uint8_t* packet, size_t length,
585 const PacketOptions& options) {
586 RtpPacket rtp_packet;
587 EXPECT_TRUE(rtp_packet.Parse(packet, length));
588 frame1_rtp_sequence_number = rtp_packet.SequenceNumber();
589 frame1_transport_sequence_number = options.packet_id;
590 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc1);
591 return true;
592 });
593 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
594 EncodedImageCallback::Result::OK);
595
596 test.AdvanceTime(TimeDelta::Millis(33));
597
598 uint16_t frame2_rtp_sequence_number = 0;
599 uint16_t frame2_transport_sequence_number = 0;
600 encoded_image.SetSpatialIndex(1);
601 EXPECT_CALL(test.transport(), SendRtp)
602 .WillOnce(
603 [&frame2_rtp_sequence_number, &frame2_transport_sequence_number](
604 const uint8_t* packet, size_t length,
605 const PacketOptions& options) {
606 RtpPacket rtp_packet;
607 EXPECT_TRUE(rtp_packet.Parse(packet, length));
608 frame2_rtp_sequence_number = rtp_packet.SequenceNumber();
609 frame2_transport_sequence_number = options.packet_id;
610 EXPECT_EQ(rtp_packet.Ssrc(), kSsrc2);
611 return true;
612 });
613 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
614 EncodedImageCallback::Result::OK);
615 test.AdvanceTime(TimeDelta::Millis(33));
616
617 EXPECT_NE(frame1_transport_sequence_number, frame2_transport_sequence_number);
618
619 // Inject a transport feedback where the packet for the first frame is lost,
620 // expect a retransmission for it.
621 EXPECT_CALL(test.transport(), SendRtp)
622 .WillOnce([&frame1_rtp_sequence_number](const uint8_t* packet,
623 size_t length,
624 const PacketOptions& options) {
625 RtpPacket rtp_packet;
626 EXPECT_TRUE(rtp_packet.Parse(packet, length));
627 EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1);
628
629 // Retransmitted sequence number from the RTX header should match
630 // the lost packet.
631 rtc::ArrayView<const uint8_t> payload = rtp_packet.payload();
632 EXPECT_EQ(ByteReader<uint16_t>::ReadBigEndian(payload.data()),
633 frame1_rtp_sequence_number);
634 return true;
635 });
636
637 StreamFeedbackObserver::StreamPacketInfo first_packet_feedback;
638 first_packet_feedback.rtp_sequence_number = frame1_rtp_sequence_number;
639 first_packet_feedback.ssrc = kSsrc1;
640 first_packet_feedback.received = false;
641
642 StreamFeedbackObserver::StreamPacketInfo second_packet_feedback;
643 second_packet_feedback.rtp_sequence_number = frame2_rtp_sequence_number;
644 second_packet_feedback.ssrc = kSsrc2;
645 second_packet_feedback.received = true;
646
647 test.router()->OnPacketFeedbackVector(
648 {first_packet_feedback, second_packet_feedback});
649
650 // Wait for pacer to run and send the RTX packet.
651 test.AdvanceTime(TimeDelta::Millis(33));
652 }
653
TEST(RtpVideoSenderTest,SupportsDependencyDescriptor)654 TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) {
655 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {});
656 test.router()->SetActive(true);
657
658 RtpHeaderExtensionMap extensions;
659 extensions.Register<RtpDependencyDescriptorExtension>(
660 kDependencyDescriptorExtensionId);
661 std::vector<RtpPacket> sent_packets;
662 ON_CALL(test.transport(), SendRtp)
663 .WillByDefault([&](const uint8_t* packet, size_t length,
664 const PacketOptions& options) {
665 sent_packets.emplace_back(&extensions);
666 EXPECT_TRUE(sent_packets.back().Parse(packet, length));
667 return true;
668 });
669
670 const uint8_t kPayload[1] = {'a'};
671 EncodedImage encoded_image;
672 encoded_image.SetTimestamp(1);
673 encoded_image.capture_time_ms_ = 2;
674 encoded_image.SetEncodedData(
675 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
676
677 CodecSpecificInfo codec_specific;
678 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
679 codec_specific.template_structure.emplace();
680 codec_specific.template_structure->num_decode_targets = 1;
681 codec_specific.template_structure->templates = {
682 FrameDependencyTemplate().T(0).Dtis("S"),
683 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}),
684 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}),
685 };
686
687 // Send two tiny images, mapping to single RTP packets.
688 // Send in key frame.
689 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
690 codec_specific.generic_frame_info =
691 GenericFrameInfo::Builder().T(0).Dtis("S").Build();
692 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}};
693 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
694 EncodedImageCallback::Result::OK);
695 test.AdvanceTime(TimeDelta::Millis(33));
696 ASSERT_THAT(sent_packets, SizeIs(1));
697 EXPECT_TRUE(
698 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
699
700 // Send in delta frame.
701 encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
702 codec_specific.template_structure = absl::nullopt;
703 codec_specific.generic_frame_info =
704 GenericFrameInfo::Builder().T(1).Dtis("D").Build();
705 codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}};
706 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
707 EncodedImageCallback::Result::OK);
708 test.AdvanceTime(TimeDelta::Millis(33));
709 ASSERT_THAT(sent_packets, SizeIs(2));
710 EXPECT_TRUE(
711 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
712 }
713
TEST(RtpVideoSenderTest,SupportsStoppingUsingDependencyDescriptor)714 TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) {
715 RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {});
716 test.router()->SetActive(true);
717
718 RtpHeaderExtensionMap extensions;
719 extensions.Register<RtpDependencyDescriptorExtension>(
720 kDependencyDescriptorExtensionId);
721 std::vector<RtpPacket> sent_packets;
722 ON_CALL(test.transport(), SendRtp)
723 .WillByDefault([&](const uint8_t* packet, size_t length,
724 const PacketOptions& options) {
725 sent_packets.emplace_back(&extensions);
726 EXPECT_TRUE(sent_packets.back().Parse(packet, length));
727 return true;
728 });
729
730 const uint8_t kPayload[1] = {'a'};
731 EncodedImage encoded_image;
732 encoded_image.SetTimestamp(1);
733 encoded_image.capture_time_ms_ = 2;
734 encoded_image.SetEncodedData(
735 EncodedImageBuffer::Create(kPayload, sizeof(kPayload)));
736
737 CodecSpecificInfo codec_specific;
738 codec_specific.codecType = VideoCodecType::kVideoCodecGeneric;
739 codec_specific.template_structure.emplace();
740 codec_specific.template_structure->num_decode_targets = 1;
741 codec_specific.template_structure->templates = {
742 FrameDependencyTemplate().T(0).Dtis("S"),
743 FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}),
744 FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}),
745 };
746
747 // Send two tiny images, mapping to single RTP packets.
748 // Send in a key frame.
749 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
750 codec_specific.generic_frame_info =
751 GenericFrameInfo::Builder().T(0).Dtis("S").Build();
752 codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}};
753 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
754 EncodedImageCallback::Result::OK);
755 test.AdvanceTime(TimeDelta::Millis(33));
756 ASSERT_THAT(sent_packets, SizeIs(1));
757 EXPECT_TRUE(
758 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
759
760 // Send in a new key frame without the support for the dependency descriptor.
761 encoded_image._frameType = VideoFrameType::kVideoFrameKey;
762 codec_specific.template_structure = absl::nullopt;
763 EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error,
764 EncodedImageCallback::Result::OK);
765 test.AdvanceTime(TimeDelta::Millis(33));
766 ASSERT_THAT(sent_packets, SizeIs(2));
767 EXPECT_FALSE(
768 sent_packets.back().HasExtension<RtpDependencyDescriptorExtension>());
769 }
770
TEST(RtpVideoSenderTest,CanSetZeroBitrateWithOverhead)771 TEST(RtpVideoSenderTest, CanSetZeroBitrateWithOverhead) {
772 test::ScopedFieldTrials trials("WebRTC-SendSideBwe-WithOverhead/Enabled/");
773 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
774 BitrateAllocationUpdate update;
775 update.target_bitrate = DataRate::Zero();
776 update.packet_loss_ratio = 0;
777 update.round_trip_time = TimeDelta::Zero();
778
779 test.router()->OnBitrateUpdated(update, /*framerate*/ 0);
780 }
781
TEST(RtpVideoSenderTest,CanSetZeroBitrateWithoutOverhead)782 TEST(RtpVideoSenderTest, CanSetZeroBitrateWithoutOverhead) {
783 RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});
784
785 BitrateAllocationUpdate update;
786 update.target_bitrate = DataRate::Zero();
787 update.packet_loss_ratio = 0;
788 update.round_trip_time = TimeDelta::Zero();
789
790 test.router()->OnBitrateUpdated(update, /*framerate*/ 0);
791 }
792
TEST(RtpVideoSenderTest,SimulcastSenderRegistersFrameTransformers)793 TEST(RtpVideoSenderTest, SimulcastSenderRegistersFrameTransformers) {
794 rtc::scoped_refptr<MockFrameTransformer> transformer =
795 new rtc::RefCountedObject<MockFrameTransformer>();
796
797 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc1));
798 EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc2));
799 RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2},
800 kPayloadType, {}, nullptr, transformer);
801
802 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc1));
803 EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc2));
804 }
805 } // namespace webrtc
806