1 /*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_video_stream_receiver.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "api/video/video_codec_type.h"
17 #include "api/video/video_frame_type.h"
18 #include "common_video/h264/h264_common.h"
19 #include "media/base/media_constants.h"
20 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
21 #include "modules/rtp_rtcp/source/rtp_format.h"
22 #include "modules/rtp_rtcp/source/rtp_format_vp9.h"
23 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
24 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
25 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
26 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
27 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
28 #include "modules/utility/include/process_thread.h"
29 #include "modules/video_coding/frame_object.h"
30 #include "modules/video_coding/include/video_coding_defines.h"
31 #include "modules/video_coding/rtp_frame_reference_finder.h"
32 #include "rtc_base/byte_buffer.h"
33 #include "rtc_base/logging.h"
34 #include "system_wrappers/include/clock.h"
35 #include "system_wrappers/include/field_trial.h"
36 #include "test/field_trial.h"
37 #include "test/gmock.h"
38 #include "test/gtest.h"
39 #include "test/mock_frame_transformer.h"
40
41 using ::testing::_;
42 using ::testing::ElementsAre;
43 using ::testing::Invoke;
44 using ::testing::SizeIs;
45 using ::testing::Values;
46
47 namespace webrtc {
48
49 namespace {
50
51 const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01};
52
GetAbsoluteCaptureTimestamps(const video_coding::EncodedFrame * frame)53 std::vector<uint64_t> GetAbsoluteCaptureTimestamps(
54 const video_coding::EncodedFrame* frame) {
55 std::vector<uint64_t> result;
56 for (const auto& packet_info : frame->PacketInfos()) {
57 if (packet_info.absolute_capture_time()) {
58 result.push_back(
59 packet_info.absolute_capture_time()->absolute_capture_timestamp);
60 }
61 }
62 return result;
63 }
64
GetGenericVideoHeader(VideoFrameType frame_type)65 RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) {
66 RTPVideoHeader video_header;
67 video_header.is_first_packet_in_frame = true;
68 video_header.is_last_packet_in_frame = true;
69 video_header.codec = kVideoCodecGeneric;
70 video_header.frame_type = frame_type;
71 return video_header;
72 }
73
74 class MockTransport : public Transport {
75 public:
76 MOCK_METHOD(bool,
77 SendRtp,
78 (const uint8_t*, size_t length, const PacketOptions& options),
79 (override));
80 MOCK_METHOD(bool, SendRtcp, (const uint8_t*, size_t length), (override));
81 };
82
83 class MockNackSender : public NackSender {
84 public:
85 MOCK_METHOD(void,
86 SendNack,
87 (const std::vector<uint16_t>& sequence_numbers,
88 bool buffering_allowed),
89 (override));
90 };
91
92 class MockKeyFrameRequestSender : public KeyFrameRequestSender {
93 public:
94 MOCK_METHOD(void, RequestKeyFrame, (), (override));
95 };
96
97 class MockOnCompleteFrameCallback
98 : public video_coding::OnCompleteFrameCallback {
99 public:
100 MOCK_METHOD(void, DoOnCompleteFrame, (video_coding::EncodedFrame*), ());
101 MOCK_METHOD(void,
102 DoOnCompleteFrameFailNullptr,
103 (video_coding::EncodedFrame*),
104 ());
105 MOCK_METHOD(void,
106 DoOnCompleteFrameFailLength,
107 (video_coding::EncodedFrame*),
108 ());
109 MOCK_METHOD(void,
110 DoOnCompleteFrameFailBitstream,
111 (video_coding::EncodedFrame*),
112 ());
OnCompleteFrame(std::unique_ptr<video_coding::EncodedFrame> frame)113 void OnCompleteFrame(
114 std::unique_ptr<video_coding::EncodedFrame> frame) override {
115 if (!frame) {
116 DoOnCompleteFrameFailNullptr(nullptr);
117 return;
118 }
119 EXPECT_EQ(buffer_.Length(), frame->size());
120 if (buffer_.Length() != frame->size()) {
121 DoOnCompleteFrameFailLength(frame.get());
122 return;
123 }
124 if (frame->size() != buffer_.Length() ||
125 memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) {
126 DoOnCompleteFrameFailBitstream(frame.get());
127 return;
128 }
129 DoOnCompleteFrame(frame.get());
130 }
131
ClearExpectedBitstream()132 void ClearExpectedBitstream() { buffer_.Clear(); }
133
AppendExpectedBitstream(const uint8_t data[],size_t size_in_bytes)134 void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) {
135 // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*.
136 buffer_.WriteBytes(reinterpret_cast<const char*>(data), size_in_bytes);
137 }
138 rtc::ByteBufferWriter buffer_;
139 };
140
141 class MockRtpPacketSink : public RtpPacketSinkInterface {
142 public:
143 MOCK_METHOD(void, OnRtpPacket, (const RtpPacketReceived&), (override));
144 };
145
146 constexpr uint32_t kSsrc = 111;
147 constexpr uint16_t kSequenceNumber = 222;
148 constexpr int kPayloadType = 100;
149 constexpr int kRedPayloadType = 125;
150
CreateRtpPacketReceived()151 std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived() {
152 auto packet = std::make_unique<RtpPacketReceived>();
153 packet->SetSsrc(kSsrc);
154 packet->SetSequenceNumber(kSequenceNumber);
155 packet->SetPayloadType(kPayloadType);
156 return packet;
157 }
158
159 MATCHER_P(SamePacketAs, other, "") {
160 return arg.Ssrc() == other.Ssrc() &&
161 arg.SequenceNumber() == other.SequenceNumber();
162 }
163
164 } // namespace
165
166 class RtpVideoStreamReceiverTest : public ::testing::Test {
167 public:
RtpVideoStreamReceiverTest()168 RtpVideoStreamReceiverTest() : RtpVideoStreamReceiverTest("") {}
RtpVideoStreamReceiverTest(std::string field_trials)169 explicit RtpVideoStreamReceiverTest(std::string field_trials)
170 : override_field_trials_(field_trials),
171 config_(CreateConfig()),
172 process_thread_(ProcessThread::Create("TestThread")) {
173 rtp_receive_statistics_ =
174 ReceiveStatistics::Create(Clock::GetRealTimeClock());
175 rtp_video_stream_receiver_ = std::make_unique<RtpVideoStreamReceiver>(
176 Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
177 rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
178 &mock_nack_sender_, &mock_key_frame_request_sender_,
179 &mock_on_complete_frame_callback_, nullptr, nullptr);
180 VideoCodec codec;
181 codec.codecType = kVideoCodecGeneric;
182 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, {},
183 /*raw_payload=*/false);
184 }
185
GetDefaultH264VideoHeader()186 RTPVideoHeader GetDefaultH264VideoHeader() {
187 RTPVideoHeader video_header;
188 video_header.codec = kVideoCodecH264;
189 video_header.video_type_header.emplace<RTPVideoHeaderH264>();
190 return video_header;
191 }
192
193 // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate
194 // code.
AddSps(RTPVideoHeader * video_header,uint8_t sps_id,rtc::CopyOnWriteBuffer * data)195 void AddSps(RTPVideoHeader* video_header,
196 uint8_t sps_id,
197 rtc::CopyOnWriteBuffer* data) {
198 NaluInfo info;
199 info.type = H264::NaluType::kSps;
200 info.sps_id = sps_id;
201 info.pps_id = -1;
202 data->AppendData<uint8_t, 2>({H264::NaluType::kSps, sps_id});
203 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
204 h264.nalus[h264.nalus_length++] = info;
205 }
206
AddPps(RTPVideoHeader * video_header,uint8_t sps_id,uint8_t pps_id,rtc::CopyOnWriteBuffer * data)207 void AddPps(RTPVideoHeader* video_header,
208 uint8_t sps_id,
209 uint8_t pps_id,
210 rtc::CopyOnWriteBuffer* data) {
211 NaluInfo info;
212 info.type = H264::NaluType::kPps;
213 info.sps_id = sps_id;
214 info.pps_id = pps_id;
215 data->AppendData<uint8_t, 2>({H264::NaluType::kPps, pps_id});
216 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
217 h264.nalus[h264.nalus_length++] = info;
218 }
219
AddIdr(RTPVideoHeader * video_header,int pps_id)220 void AddIdr(RTPVideoHeader* video_header, int pps_id) {
221 NaluInfo info;
222 info.type = H264::NaluType::kIdr;
223 info.sps_id = -1;
224 info.pps_id = pps_id;
225 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
226 h264.nalus[h264.nalus_length++] = info;
227 }
228
229 protected:
CreateConfig()230 static VideoReceiveStream::Config CreateConfig() {
231 VideoReceiveStream::Config config(nullptr);
232 config.rtp.remote_ssrc = 1111;
233 config.rtp.local_ssrc = 2222;
234 config.rtp.red_payload_type = kRedPayloadType;
235 return config;
236 }
237
238 const webrtc::test::ScopedFieldTrials override_field_trials_;
239 VideoReceiveStream::Config config_;
240 MockNackSender mock_nack_sender_;
241 MockKeyFrameRequestSender mock_key_frame_request_sender_;
242 MockTransport mock_transport_;
243 MockOnCompleteFrameCallback mock_on_complete_frame_callback_;
244 std::unique_ptr<ProcessThread> process_thread_;
245 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
246 std::unique_ptr<RtpVideoStreamReceiver> rtp_video_stream_receiver_;
247 };
248
TEST_F(RtpVideoStreamReceiverTest,CacheColorSpaceFromLastPacketOfKeyframe)249 TEST_F(RtpVideoStreamReceiverTest, CacheColorSpaceFromLastPacketOfKeyframe) {
250 // Test that color space is cached from the last packet of a key frame and
251 // that it's not reset by padding packets without color space.
252 constexpr int kVp9PayloadType = 99;
253 const ColorSpace kColorSpace(
254 ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12,
255 ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull);
256 const std::vector<uint8_t> kKeyFramePayload = {0, 1, 2, 3, 4, 5,
257 6, 7, 8, 9, 10};
258 const std::vector<uint8_t> kDeltaFramePayload = {0, 1, 2, 3, 4};
259
260 // Anonymous helper class that generates received packets.
261 class {
262 public:
263 void SetPayload(const std::vector<uint8_t>& payload,
264 VideoFrameType video_frame_type) {
265 video_frame_type_ = video_frame_type;
266 RtpPacketizer::PayloadSizeLimits pay_load_size_limits;
267 // Reduce max payload length to make sure the key frame generates two
268 // packets.
269 pay_load_size_limits.max_payload_len = 8;
270 RTPVideoHeaderVP9 rtp_video_header_vp9;
271 rtp_video_header_vp9.InitRTPVideoHeaderVP9();
272 rtp_video_header_vp9.inter_pic_predicted =
273 (video_frame_type == VideoFrameType::kVideoFrameDelta);
274 rtp_packetizer_ = std::make_unique<RtpPacketizerVp9>(
275 payload, pay_load_size_limits, rtp_video_header_vp9);
276 }
277
278 size_t NumPackets() { return rtp_packetizer_->NumPackets(); }
279 void SetColorSpace(const ColorSpace& color_space) {
280 color_space_ = color_space;
281 }
282
283 RtpPacketReceived NextPacket() {
284 RtpHeaderExtensionMap extension_map;
285 extension_map.Register<ColorSpaceExtension>(1);
286 RtpPacketToSend packet_to_send(&extension_map);
287 packet_to_send.SetSequenceNumber(sequence_number_++);
288 packet_to_send.SetSsrc(kSsrc);
289 packet_to_send.SetPayloadType(kVp9PayloadType);
290 bool include_color_space =
291 (rtp_packetizer_->NumPackets() == 1u &&
292 video_frame_type_ == VideoFrameType::kVideoFrameKey);
293 if (include_color_space) {
294 EXPECT_TRUE(
295 packet_to_send.SetExtension<ColorSpaceExtension>(color_space_));
296 }
297 rtp_packetizer_->NextPacket(&packet_to_send);
298
299 RtpPacketReceived received_packet(&extension_map);
300 received_packet.Parse(packet_to_send.data(), packet_to_send.size());
301 return received_packet;
302 }
303
304 private:
305 uint16_t sequence_number_ = 0;
306 VideoFrameType video_frame_type_;
307 ColorSpace color_space_;
308 std::unique_ptr<RtpPacketizer> rtp_packetizer_;
309 } received_packet_generator;
310 received_packet_generator.SetColorSpace(kColorSpace);
311
312 // Prepare the receiver for VP9.
313 VideoCodec codec;
314 codec.codecType = kVideoCodecVP9;
315 std::map<std::string, std::string> codec_params;
316 rtp_video_stream_receiver_->AddReceiveCodec(kVp9PayloadType, codec,
317 codec_params,
318 /*raw_payload=*/false);
319
320 // Generate key frame packets.
321 received_packet_generator.SetPayload(kKeyFramePayload,
322 VideoFrameType::kVideoFrameKey);
323 EXPECT_EQ(received_packet_generator.NumPackets(), 2u);
324 RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket();
325 RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket();
326
327 // Generate delta frame packet.
328 received_packet_generator.SetPayload(kDeltaFramePayload,
329 VideoFrameType::kVideoFrameDelta);
330 EXPECT_EQ(received_packet_generator.NumPackets(), 1u);
331 RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket();
332
333 rtp_video_stream_receiver_->StartReceive();
334 mock_on_complete_frame_callback_.AppendExpectedBitstream(
335 kKeyFramePayload.data(), kKeyFramePayload.size());
336
337 // Send the key frame and expect a callback with color space information.
338 EXPECT_FALSE(key_frame_packet1.GetExtension<ColorSpaceExtension>());
339 EXPECT_TRUE(key_frame_packet2.GetExtension<ColorSpaceExtension>());
340 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
341 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
342 .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) {
343 ASSERT_TRUE(frame->EncodedImage().ColorSpace());
344 EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
345 }));
346 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2);
347 // Resend the first key frame packet to simulate padding for example.
348 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
349
350 mock_on_complete_frame_callback_.ClearExpectedBitstream();
351 mock_on_complete_frame_callback_.AppendExpectedBitstream(
352 kDeltaFramePayload.data(), kDeltaFramePayload.size());
353
354 // Expect delta frame to have color space set even though color space not
355 // included in the RTP packet.
356 EXPECT_FALSE(delta_frame_packet.GetExtension<ColorSpaceExtension>());
357 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
358 .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) {
359 ASSERT_TRUE(frame->EncodedImage().ColorSpace());
360 EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
361 }));
362 rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet);
363 }
364
TEST_F(RtpVideoStreamReceiverTest,GenericKeyFrame)365 TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrame) {
366 RtpPacketReceived rtp_packet;
367 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
368 rtp_packet.SetPayloadType(kPayloadType);
369 rtp_packet.SetSequenceNumber(1);
370 RTPVideoHeader video_header =
371 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
372 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
373 data.size());
374 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
375 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
376 video_header);
377 }
378
TEST_F(RtpVideoStreamReceiverTest,PacketInfoIsPropagatedIntoVideoFrames)379 TEST_F(RtpVideoStreamReceiverTest, PacketInfoIsPropagatedIntoVideoFrames) {
380 constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
381 constexpr int kId0 = 1;
382
383 RtpHeaderExtensionMap extension_map;
384 extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
385 RtpPacketReceived rtp_packet(&extension_map);
386 rtp_packet.SetPayloadType(kPayloadType);
387 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
388 rtp_packet.SetSequenceNumber(1);
389 rtp_packet.SetTimestamp(1);
390 rtp_packet.SetSsrc(kSsrc);
391 rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
392 AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
393 /*estimated_capture_clock_offset=*/absl::nullopt});
394
395 RTPVideoHeader video_header =
396 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
397 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
398 data.size());
399 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
400 .WillOnce(Invoke(
401 [kAbsoluteCaptureTimestamp](video_coding::EncodedFrame* frame) {
402 EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame),
403 ElementsAre(kAbsoluteCaptureTimestamp));
404 }));
405 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
406 video_header);
407 }
408
TEST_F(RtpVideoStreamReceiverTest,MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue)409 TEST_F(RtpVideoStreamReceiverTest,
410 MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) {
411 constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
412 constexpr int kId0 = 1;
413
414 RtpHeaderExtensionMap extension_map;
415 extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
416 RtpPacketReceived rtp_packet(&extension_map);
417 rtp_packet.SetPayloadType(kPayloadType);
418
419 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
420 uint16_t sequence_number = 1;
421 uint32_t rtp_timestamp = 1;
422 rtp_packet.SetSequenceNumber(sequence_number);
423 rtp_packet.SetTimestamp(rtp_timestamp);
424 rtp_packet.SetSsrc(kSsrc);
425 rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
426 AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
427 /*estimated_capture_clock_offset=*/absl::nullopt});
428
429 RTPVideoHeader video_header =
430 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
431 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
432 data.size());
433 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
434 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
435 video_header);
436
437 // Rtp packet without absolute capture time.
438 rtp_packet = RtpPacketReceived(&extension_map);
439 rtp_packet.SetPayloadType(kPayloadType);
440 rtp_packet.SetSequenceNumber(++sequence_number);
441 rtp_packet.SetTimestamp(++rtp_timestamp);
442 rtp_packet.SetSsrc(kSsrc);
443
444 // There is no absolute capture time in the second packet.
445 // Expect rtp video stream receiver to extrapolate it for the resulting video
446 // frame using absolute capture time from the previous packet.
447 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
448 .WillOnce(Invoke([](video_coding::EncodedFrame* frame) {
449 EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1));
450 }));
451 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
452 video_header);
453 }
454
TEST_F(RtpVideoStreamReceiverTest,NoInfiniteRecursionOnEncapsulatedRedPacket)455 TEST_F(RtpVideoStreamReceiverTest, NoInfiniteRecursionOnEncapsulatedRedPacket) {
456 const std::vector<uint8_t> data({
457 0x80, // RTP version.
458 kRedPayloadType, // Payload type.
459 0, 0, 0, 0, 0, 0, // Don't care.
460 0, 0, 0x4, 0x57, // SSRC
461 kRedPayloadType, // RED header.
462 0, 0, 0, 0, 0 // Don't care.
463 });
464 RtpPacketReceived packet;
465 EXPECT_TRUE(packet.Parse(data.data(), data.size()));
466 rtp_video_stream_receiver_->StartReceive();
467 rtp_video_stream_receiver_->OnRtpPacket(packet);
468 }
469
TEST_F(RtpVideoStreamReceiverTest,DropsPacketWithRedPayloadTypeAndEmptyPayload)470 TEST_F(RtpVideoStreamReceiverTest,
471 DropsPacketWithRedPayloadTypeAndEmptyPayload) {
472 const uint8_t kRedPayloadType = 125;
473 config_.rtp.red_payload_type = kRedPayloadType;
474 SetUp(); // re-create rtp_video_stream_receiver with red payload type.
475 // clang-format off
476 const uint8_t data[] = {
477 0x80, // RTP version.
478 kRedPayloadType, // Payload type.
479 0, 0, 0, 0, 0, 0, // Don't care.
480 0, 0, 0x4, 0x57, // SSRC
481 // Empty rtp payload.
482 };
483 // clang-format on
484 RtpPacketReceived packet;
485 // Manually convert to CopyOnWriteBuffer to be sure capacity == size
486 // and asan bot can catch read buffer overflow.
487 EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data)));
488 rtp_video_stream_receiver_->StartReceive();
489 rtp_video_stream_receiver_->OnRtpPacket(packet);
490 // Expect asan doesn't find anything.
491 }
492
TEST_F(RtpVideoStreamReceiverTest,GenericKeyFrameBitstreamError)493 TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrameBitstreamError) {
494 RtpPacketReceived rtp_packet;
495 rtp_packet.SetPayloadType(kPayloadType);
496 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
497 rtp_packet.SetSequenceNumber(1);
498 RTPVideoHeader video_header =
499 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
500 constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff};
501 mock_on_complete_frame_callback_.AppendExpectedBitstream(
502 expected_bitsteam, sizeof(expected_bitsteam));
503 EXPECT_CALL(mock_on_complete_frame_callback_,
504 DoOnCompleteFrameFailBitstream(_));
505 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
506 video_header);
507 }
508
509 class RtpVideoStreamReceiverTestH264
510 : public RtpVideoStreamReceiverTest,
511 public ::testing::WithParamInterface<std::string> {
512 protected:
RtpVideoStreamReceiverTestH264()513 RtpVideoStreamReceiverTestH264() : RtpVideoStreamReceiverTest(GetParam()) {}
514 };
515
516 INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe,
517 RtpVideoStreamReceiverTestH264,
518 Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/"));
519
TEST_P(RtpVideoStreamReceiverTestH264,InBandSpsPps)520 TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) {
521 rtc::CopyOnWriteBuffer sps_data;
522 RtpPacketReceived rtp_packet;
523 RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
524 AddSps(&sps_video_header, 0, &sps_data);
525 rtp_packet.SetSequenceNumber(0);
526 rtp_packet.SetPayloadType(kPayloadType);
527 sps_video_header.is_first_packet_in_frame = true;
528 sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
529 mock_on_complete_frame_callback_.AppendExpectedBitstream(
530 kH264StartCode, sizeof(kH264StartCode));
531 mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
532 sps_data.size());
533 rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
534 sps_video_header);
535
536 rtc::CopyOnWriteBuffer pps_data;
537 RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
538 AddPps(&pps_video_header, 0, 1, &pps_data);
539 rtp_packet.SetSequenceNumber(1);
540 pps_video_header.is_first_packet_in_frame = true;
541 pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
542 mock_on_complete_frame_callback_.AppendExpectedBitstream(
543 kH264StartCode, sizeof(kH264StartCode));
544 mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
545 pps_data.size());
546 rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
547 pps_video_header);
548
549 rtc::CopyOnWriteBuffer idr_data;
550 RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
551 AddIdr(&idr_video_header, 1);
552 rtp_packet.SetSequenceNumber(2);
553 idr_video_header.is_first_packet_in_frame = true;
554 idr_video_header.is_last_packet_in_frame = true;
555 idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
556 const uint8_t idr[] = {0x65, 1, 2, 3};
557 idr_data.AppendData(idr);
558 mock_on_complete_frame_callback_.AppendExpectedBitstream(
559 kH264StartCode, sizeof(kH264StartCode));
560 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
561 idr_data.size());
562 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
563 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
564 idr_video_header);
565 }
566
TEST_P(RtpVideoStreamReceiverTestH264,OutOfBandFmtpSpsPps)567 TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) {
568 constexpr int kPayloadType = 99;
569 VideoCodec codec;
570 std::map<std::string, std::string> codec_params;
571 // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2
572 // .
573 codec_params.insert(
574 {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="});
575 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, codec_params,
576 /*raw_payload=*/false);
577 const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96,
578 0x53, 0x05, 0x89, 0x88};
579 mock_on_complete_frame_callback_.AppendExpectedBitstream(
580 kH264StartCode, sizeof(kH264StartCode));
581 mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps,
582 sizeof(binary_sps));
583 const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88};
584 mock_on_complete_frame_callback_.AppendExpectedBitstream(
585 kH264StartCode, sizeof(kH264StartCode));
586 mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps,
587 sizeof(binary_pps));
588
589 RtpPacketReceived rtp_packet;
590 RTPVideoHeader video_header = GetDefaultH264VideoHeader();
591 AddIdr(&video_header, 0);
592 rtp_packet.SetPayloadType(kPayloadType);
593 rtp_packet.SetSequenceNumber(2);
594 video_header.is_first_packet_in_frame = true;
595 video_header.is_last_packet_in_frame = true;
596 video_header.codec = kVideoCodecH264;
597 video_header.frame_type = VideoFrameType::kVideoFrameKey;
598 rtc::CopyOnWriteBuffer data({1, 2, 3});
599 mock_on_complete_frame_callback_.AppendExpectedBitstream(
600 kH264StartCode, sizeof(kH264StartCode));
601 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
602 data.size());
603 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
604 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
605 video_header);
606 }
607
TEST_P(RtpVideoStreamReceiverTestH264,ForceSpsPpsIdrIsKeyframe)608 TEST_P(RtpVideoStreamReceiverTestH264, ForceSpsPpsIdrIsKeyframe) {
609 constexpr int kPayloadType = 99;
610 VideoCodec codec;
611 std::map<std::string, std::string> codec_params;
612 if (GetParam() ==
613 "") { // Forcing can be done either with field trial or codec_params.
614 codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""});
615 }
616 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, codec_params,
617 /*raw_payload=*/false);
618 rtc::CopyOnWriteBuffer sps_data;
619 RtpPacketReceived rtp_packet;
620 RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
621 AddSps(&sps_video_header, 0, &sps_data);
622 rtp_packet.SetSequenceNumber(0);
623 rtp_packet.SetPayloadType(kPayloadType);
624 sps_video_header.is_first_packet_in_frame = true;
625 sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
626 mock_on_complete_frame_callback_.AppendExpectedBitstream(
627 kH264StartCode, sizeof(kH264StartCode));
628 mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
629 sps_data.size());
630 rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
631 sps_video_header);
632
633 rtc::CopyOnWriteBuffer pps_data;
634 RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
635 AddPps(&pps_video_header, 0, 1, &pps_data);
636 rtp_packet.SetSequenceNumber(1);
637 pps_video_header.is_first_packet_in_frame = true;
638 pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
639 mock_on_complete_frame_callback_.AppendExpectedBitstream(
640 kH264StartCode, sizeof(kH264StartCode));
641 mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
642 pps_data.size());
643 rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
644 pps_video_header);
645
646 rtc::CopyOnWriteBuffer idr_data;
647 RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
648 AddIdr(&idr_video_header, 1);
649 rtp_packet.SetSequenceNumber(2);
650 idr_video_header.is_first_packet_in_frame = true;
651 idr_video_header.is_last_packet_in_frame = true;
652 idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
653 const uint8_t idr[] = {0x65, 1, 2, 3};
654 idr_data.AppendData(idr);
655 mock_on_complete_frame_callback_.AppendExpectedBitstream(
656 kH264StartCode, sizeof(kH264StartCode));
657 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
658 idr_data.size());
659 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
660 .WillOnce([&](video_coding::EncodedFrame* frame) {
661 EXPECT_TRUE(frame->is_keyframe());
662 });
663 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
664 idr_video_header);
665 mock_on_complete_frame_callback_.ClearExpectedBitstream();
666 mock_on_complete_frame_callback_.AppendExpectedBitstream(
667 kH264StartCode, sizeof(kH264StartCode));
668 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
669 idr_data.size());
670 rtp_packet.SetSequenceNumber(3);
671 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
672 .WillOnce([&](video_coding::EncodedFrame* frame) {
673 EXPECT_FALSE(frame->is_keyframe());
674 });
675 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
676 idr_video_header);
677 }
678
TEST_F(RtpVideoStreamReceiverTest,PaddingInMediaStream)679 TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) {
680 RtpPacketReceived rtp_packet;
681 RTPVideoHeader video_header = GetDefaultH264VideoHeader();
682 rtc::CopyOnWriteBuffer data({1, 2, 3});
683 rtp_packet.SetPayloadType(kPayloadType);
684 rtp_packet.SetSequenceNumber(2);
685 video_header.is_first_packet_in_frame = true;
686 video_header.is_last_packet_in_frame = true;
687 video_header.codec = kVideoCodecGeneric;
688 video_header.frame_type = VideoFrameType::kVideoFrameKey;
689 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
690 data.size());
691
692 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
693 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
694 video_header);
695
696 rtp_packet.SetSequenceNumber(3);
697 rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
698 video_header);
699
700 rtp_packet.SetSequenceNumber(4);
701 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
702 video_header.frame_type = VideoFrameType::kVideoFrameDelta;
703 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
704 video_header);
705
706 rtp_packet.SetSequenceNumber(6);
707 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
708 video_header);
709
710 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
711 rtp_packet.SetSequenceNumber(5);
712 rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
713 video_header);
714 }
715
TEST_F(RtpVideoStreamReceiverTest,RequestKeyframeIfFirstFrameIsDelta)716 TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeIfFirstFrameIsDelta) {
717 RtpPacketReceived rtp_packet;
718 rtp_packet.SetPayloadType(kPayloadType);
719 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
720 rtp_packet.SetSequenceNumber(1);
721 RTPVideoHeader video_header =
722 GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
723 EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
724 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
725 video_header);
726 }
727
TEST_F(RtpVideoStreamReceiverTest,RequestKeyframeWhenPacketBufferGetsFull)728 TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeWhenPacketBufferGetsFull) {
729 constexpr int kPacketBufferMaxSize = 2048;
730
731 RtpPacketReceived rtp_packet;
732 rtp_packet.SetPayloadType(kPayloadType);
733 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
734 RTPVideoHeader video_header =
735 GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
736 // Incomplete frames so that the packet buffer is filling up.
737 video_header.is_last_packet_in_frame = false;
738 uint16_t start_sequence_number = 1234;
739 rtp_packet.SetSequenceNumber(start_sequence_number);
740 while (rtp_packet.SequenceNumber() - start_sequence_number <
741 kPacketBufferMaxSize) {
742 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
743 video_header);
744 rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2);
745 }
746
747 EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
748 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
749 video_header);
750 }
751
TEST_F(RtpVideoStreamReceiverTest,SecondarySinksGetRtpNotifications)752 TEST_F(RtpVideoStreamReceiverTest, SecondarySinksGetRtpNotifications) {
753 rtp_video_stream_receiver_->StartReceive();
754
755 MockRtpPacketSink secondary_sink_1;
756 MockRtpPacketSink secondary_sink_2;
757
758 rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1);
759 rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2);
760
761 auto rtp_packet = CreateRtpPacketReceived();
762 EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet)));
763 EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet)));
764
765 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
766
767 // Test tear-down.
768 rtp_video_stream_receiver_->StopReceive();
769 rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1);
770 rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2);
771 }
772
TEST_F(RtpVideoStreamReceiverTest,RemovedSecondarySinksGetNoRtpNotifications)773 TEST_F(RtpVideoStreamReceiverTest, RemovedSecondarySinksGetNoRtpNotifications) {
774 rtp_video_stream_receiver_->StartReceive();
775
776 MockRtpPacketSink secondary_sink;
777
778 rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
779 rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
780
781 auto rtp_packet = CreateRtpPacketReceived();
782
783 EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
784
785 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
786
787 // Test tear-down.
788 rtp_video_stream_receiver_->StopReceive();
789 }
790
TEST_F(RtpVideoStreamReceiverTest,OnlyRemovedSecondarySinksExcludedFromNotifications)791 TEST_F(RtpVideoStreamReceiverTest,
792 OnlyRemovedSecondarySinksExcludedFromNotifications) {
793 rtp_video_stream_receiver_->StartReceive();
794
795 MockRtpPacketSink kept_secondary_sink;
796 MockRtpPacketSink removed_secondary_sink;
797
798 rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink);
799 rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink);
800 rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink);
801
802 auto rtp_packet = CreateRtpPacketReceived();
803 EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet)));
804
805 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
806
807 // Test tear-down.
808 rtp_video_stream_receiver_->StopReceive();
809 rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink);
810 }
811
TEST_F(RtpVideoStreamReceiverTest,SecondariesOfNonStartedStreamGetNoNotifications)812 TEST_F(RtpVideoStreamReceiverTest,
813 SecondariesOfNonStartedStreamGetNoNotifications) {
814 // Explicitly showing that the stream is not in the |started| state,
815 // regardless of whether streams start out |started| or |stopped|.
816 rtp_video_stream_receiver_->StopReceive();
817
818 MockRtpPacketSink secondary_sink;
819 rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
820
821 auto rtp_packet = CreateRtpPacketReceived();
822 EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0);
823
824 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
825
826 // Test tear-down.
827 rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
828 }
829
TEST_F(RtpVideoStreamReceiverTest,ParseGenericDescriptorOnePacket)830 TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorOnePacket) {
831 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
832 const int kSpatialIndex = 1;
833
834 rtp_video_stream_receiver_->StartReceive();
835
836 RtpHeaderExtensionMap extension_map;
837 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
838 RtpPacketReceived rtp_packet(&extension_map);
839 rtp_packet.SetPayloadType(kPayloadType);
840
841 RtpGenericFrameDescriptor generic_descriptor;
842 generic_descriptor.SetFirstPacketInSubFrame(true);
843 generic_descriptor.SetLastPacketInSubFrame(true);
844 generic_descriptor.SetFrameId(100);
845 generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
846 generic_descriptor.AddFrameDependencyDiff(90);
847 generic_descriptor.AddFrameDependencyDiff(80);
848 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
849 generic_descriptor));
850
851 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
852 memcpy(payload, data.data(), data.size());
853 // The first byte is the header, so we ignore the first byte of |data|.
854 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
855 data.size() - 1);
856
857 rtp_packet.SetMarker(true);
858 rtp_packet.SetPayloadType(kPayloadType);
859 rtp_packet.SetSequenceNumber(1);
860
861 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
862 .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) {
863 EXPECT_EQ(frame->num_references, 2U);
864 EXPECT_EQ(frame->references[0], frame->id.picture_id - 90);
865 EXPECT_EQ(frame->references[1], frame->id.picture_id - 80);
866 EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex);
867 EXPECT_THAT(frame->PacketInfos(), SizeIs(1));
868 }));
869
870 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
871 }
872
TEST_F(RtpVideoStreamReceiverTest,ParseGenericDescriptorTwoPackets)873 TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorTwoPackets) {
874 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
875 const int kSpatialIndex = 1;
876
877 rtp_video_stream_receiver_->StartReceive();
878
879 RtpHeaderExtensionMap extension_map;
880 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
881 RtpPacketReceived first_packet(&extension_map);
882
883 RtpGenericFrameDescriptor first_packet_descriptor;
884 first_packet_descriptor.SetFirstPacketInSubFrame(true);
885 first_packet_descriptor.SetLastPacketInSubFrame(false);
886 first_packet_descriptor.SetFrameId(100);
887 first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
888 first_packet_descriptor.SetResolution(480, 360);
889 ASSERT_TRUE(first_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
890 first_packet_descriptor));
891
892 uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size());
893 memcpy(first_packet_payload, data.data(), data.size());
894 // The first byte is the header, so we ignore the first byte of |data|.
895 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
896 data.size() - 1);
897
898 first_packet.SetPayloadType(kPayloadType);
899 first_packet.SetSequenceNumber(1);
900 rtp_video_stream_receiver_->OnRtpPacket(first_packet);
901
902 RtpPacketReceived second_packet(&extension_map);
903 RtpGenericFrameDescriptor second_packet_descriptor;
904 second_packet_descriptor.SetFirstPacketInSubFrame(false);
905 second_packet_descriptor.SetLastPacketInSubFrame(true);
906 ASSERT_TRUE(second_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
907 second_packet_descriptor));
908
909 second_packet.SetMarker(true);
910 second_packet.SetPayloadType(kPayloadType);
911 second_packet.SetSequenceNumber(2);
912
913 uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size());
914 memcpy(second_packet_payload, data.data(), data.size());
915 // The first byte is the header, so we ignore the first byte of |data|.
916 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
917 data.size() - 1);
918
919 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
920 .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) {
921 EXPECT_EQ(frame->num_references, 0U);
922 EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex);
923 EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u);
924 EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u);
925 EXPECT_THAT(frame->PacketInfos(), SizeIs(2));
926 }));
927
928 rtp_video_stream_receiver_->OnRtpPacket(second_packet);
929 }
930
TEST_F(RtpVideoStreamReceiverTest,ParseGenericDescriptorRawPayload)931 TEST_F(RtpVideoStreamReceiverTest, ParseGenericDescriptorRawPayload) {
932 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
933 const int kRawPayloadType = 123;
934
935 VideoCodec codec;
936 rtp_video_stream_receiver_->AddReceiveCodec(kRawPayloadType, codec, {},
937 /*raw_payload=*/true);
938 rtp_video_stream_receiver_->StartReceive();
939
940 RtpHeaderExtensionMap extension_map;
941 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
942 RtpPacketReceived rtp_packet(&extension_map);
943
944 RtpGenericFrameDescriptor generic_descriptor;
945 generic_descriptor.SetFirstPacketInSubFrame(true);
946 generic_descriptor.SetLastPacketInSubFrame(true);
947 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
948 generic_descriptor));
949
950 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
951 memcpy(payload, data.data(), data.size());
952 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
953 data.size());
954
955 rtp_packet.SetMarker(true);
956 rtp_packet.SetPayloadType(kRawPayloadType);
957 rtp_packet.SetSequenceNumber(1);
958
959 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
960 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
961 }
962
TEST_F(RtpVideoStreamReceiverTest,UnwrapsFrameId)963 TEST_F(RtpVideoStreamReceiverTest, UnwrapsFrameId) {
964 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
965 const int kPayloadType = 123;
966
967 VideoCodec codec;
968 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, {},
969 /*raw_payload=*/true);
970 rtp_video_stream_receiver_->StartReceive();
971 RtpHeaderExtensionMap extension_map;
972 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
973
974 uint16_t rtp_sequence_number = 1;
975 auto inject_packet = [&](uint16_t wrapped_frame_id) {
976 RtpPacketReceived rtp_packet(&extension_map);
977
978 RtpGenericFrameDescriptor generic_descriptor;
979 generic_descriptor.SetFirstPacketInSubFrame(true);
980 generic_descriptor.SetLastPacketInSubFrame(true);
981 generic_descriptor.SetFrameId(wrapped_frame_id);
982 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
983 generic_descriptor));
984
985 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
986 ASSERT_TRUE(payload);
987 memcpy(payload, data.data(), data.size());
988 mock_on_complete_frame_callback_.ClearExpectedBitstream();
989 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
990 data.size());
991 rtp_packet.SetMarker(true);
992 rtp_packet.SetPayloadType(kPayloadType);
993 rtp_packet.SetSequenceNumber(++rtp_sequence_number);
994 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
995 };
996
997 int64_t first_picture_id;
998 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
999 .WillOnce([&](video_coding::EncodedFrame* frame) {
1000 first_picture_id = frame->id.picture_id;
1001 });
1002 inject_packet(/*wrapped_frame_id=*/0xffff);
1003
1004 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1005 .WillOnce([&](video_coding::EncodedFrame* frame) {
1006 EXPECT_EQ(frame->id.picture_id - first_picture_id, 3);
1007 });
1008 inject_packet(/*wrapped_frame_id=*/0x0002);
1009 }
1010
1011 class RtpVideoStreamReceiverDependencyDescriptorTest
1012 : public RtpVideoStreamReceiverTest {
1013 public:
RtpVideoStreamReceiverDependencyDescriptorTest()1014 RtpVideoStreamReceiverDependencyDescriptorTest() {
1015 VideoCodec codec;
1016 rtp_video_stream_receiver_->AddReceiveCodec(payload_type_, codec, {},
1017 /*raw_payload=*/true);
1018 extension_map_.Register<RtpDependencyDescriptorExtension>(7);
1019 rtp_video_stream_receiver_->StartReceive();
1020 }
1021
1022 // Returns some valid structure for the DependencyDescriptors.
1023 // First template of that structure always fit for a key frame.
CreateStreamStructure()1024 static FrameDependencyStructure CreateStreamStructure() {
1025 FrameDependencyStructure stream_structure;
1026 stream_structure.num_decode_targets = 1;
1027 stream_structure.templates = {
1028 FrameDependencyTemplate().Dtis("S"),
1029 FrameDependencyTemplate().Dtis("S").FrameDiffs({1}),
1030 };
1031 return stream_structure;
1032 }
1033
InjectPacketWith(const FrameDependencyStructure & stream_structure,const DependencyDescriptor & dependency_descriptor)1034 void InjectPacketWith(const FrameDependencyStructure& stream_structure,
1035 const DependencyDescriptor& dependency_descriptor) {
1036 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
1037 RtpPacketReceived rtp_packet(&extension_map_);
1038 ASSERT_TRUE(rtp_packet.SetExtension<RtpDependencyDescriptorExtension>(
1039 stream_structure, dependency_descriptor));
1040 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
1041 ASSERT_TRUE(payload);
1042 memcpy(payload, data.data(), data.size());
1043 mock_on_complete_frame_callback_.ClearExpectedBitstream();
1044 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
1045 data.size());
1046 rtp_packet.SetMarker(true);
1047 rtp_packet.SetPayloadType(payload_type_);
1048 rtp_packet.SetSequenceNumber(++rtp_sequence_number_);
1049 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
1050 }
1051
1052 private:
1053 const int payload_type_ = 123;
1054 RtpHeaderExtensionMap extension_map_;
1055 uint16_t rtp_sequence_number_ = 321;
1056 };
1057
TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,UnwrapsFrameId)1058 TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest, UnwrapsFrameId) {
1059 FrameDependencyStructure stream_structure = CreateStreamStructure();
1060
1061 DependencyDescriptor keyframe_descriptor;
1062 keyframe_descriptor.attached_structure =
1063 std::make_unique<FrameDependencyStructure>(stream_structure);
1064 keyframe_descriptor.frame_dependencies = stream_structure.templates[0];
1065 keyframe_descriptor.frame_number = 0xfff0;
1066 // DependencyDescriptor doesn't support reordering delta frame before
1067 // keyframe. Thus feed a key frame first, then test reodered delta frames.
1068 int64_t first_picture_id;
1069 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1070 .WillOnce([&](video_coding::EncodedFrame* frame) {
1071 first_picture_id = frame->id.picture_id;
1072 });
1073 InjectPacketWith(stream_structure, keyframe_descriptor);
1074
1075 DependencyDescriptor deltaframe1_descriptor;
1076 deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
1077 deltaframe1_descriptor.frame_number = 0xfffe;
1078
1079 DependencyDescriptor deltaframe2_descriptor;
1080 deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
1081 deltaframe2_descriptor.frame_number = 0x0002;
1082
1083 // Parser should unwrap frame ids correctly even if packets were reordered by
1084 // the network.
1085 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1086 .WillOnce([&](video_coding::EncodedFrame* frame) {
1087 // 0x0002 - 0xfff0
1088 EXPECT_EQ(frame->id.picture_id - first_picture_id, 18);
1089 })
1090 .WillOnce([&](video_coding::EncodedFrame* frame) {
1091 // 0xfffe - 0xfff0
1092 EXPECT_EQ(frame->id.picture_id - first_picture_id, 14);
1093 });
1094 InjectPacketWith(stream_structure, deltaframe2_descriptor);
1095 InjectPacketWith(stream_structure, deltaframe1_descriptor);
1096 }
1097
TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,DropsLateDeltaFramePacketWithDependencyDescriptorExtension)1098 TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,
1099 DropsLateDeltaFramePacketWithDependencyDescriptorExtension) {
1100 FrameDependencyStructure stream_structure1 = CreateStreamStructure();
1101 FrameDependencyStructure stream_structure2 = CreateStreamStructure();
1102 // Make sure template ids for these two structures do not collide:
1103 // adjust structure_id (that is also used as template id offset).
1104 stream_structure1.structure_id = 13;
1105 stream_structure2.structure_id =
1106 stream_structure1.structure_id + stream_structure1.templates.size();
1107
1108 DependencyDescriptor keyframe1_descriptor;
1109 keyframe1_descriptor.attached_structure =
1110 std::make_unique<FrameDependencyStructure>(stream_structure1);
1111 keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
1112 keyframe1_descriptor.frame_number = 1;
1113 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
1114 InjectPacketWith(stream_structure1, keyframe1_descriptor);
1115
1116 // Pass in 2nd key frame with different structure.
1117 DependencyDescriptor keyframe2_descriptor;
1118 keyframe2_descriptor.attached_structure =
1119 std::make_unique<FrameDependencyStructure>(stream_structure2);
1120 keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
1121 keyframe2_descriptor.frame_number = 3;
1122 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
1123 InjectPacketWith(stream_structure2, keyframe2_descriptor);
1124
1125 // Pass in late delta frame that uses structure of the 1st key frame.
1126 DependencyDescriptor deltaframe_descriptor;
1127 deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0];
1128 deltaframe_descriptor.frame_number = 2;
1129 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0);
1130 InjectPacketWith(stream_structure1, deltaframe_descriptor);
1131 }
1132
TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,DropsLateKeyFramePacketWithDependencyDescriptorExtension)1133 TEST_F(RtpVideoStreamReceiverDependencyDescriptorTest,
1134 DropsLateKeyFramePacketWithDependencyDescriptorExtension) {
1135 FrameDependencyStructure stream_structure1 = CreateStreamStructure();
1136 FrameDependencyStructure stream_structure2 = CreateStreamStructure();
1137 // Make sure template ids for these two structures do not collide:
1138 // adjust structure_id (that is also used as template id offset).
1139 stream_structure1.structure_id = 13;
1140 stream_structure2.structure_id =
1141 stream_structure1.structure_id + stream_structure1.templates.size();
1142
1143 DependencyDescriptor keyframe1_descriptor;
1144 keyframe1_descriptor.attached_structure =
1145 std::make_unique<FrameDependencyStructure>(stream_structure1);
1146 keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
1147 keyframe1_descriptor.frame_number = 1;
1148
1149 DependencyDescriptor keyframe2_descriptor;
1150 keyframe2_descriptor.attached_structure =
1151 std::make_unique<FrameDependencyStructure>(stream_structure2);
1152 keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
1153 keyframe2_descriptor.frame_number = 3;
1154
1155 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1156 .WillOnce([&](video_coding::EncodedFrame* frame) {
1157 EXPECT_EQ(frame->id.picture_id & 0xFFFF, 3);
1158 });
1159 InjectPacketWith(stream_structure2, keyframe2_descriptor);
1160 InjectPacketWith(stream_structure1, keyframe1_descriptor);
1161
1162 // Pass in delta frame that uses structure of the 2nd key frame. Late key
1163 // frame shouldn't block it.
1164 DependencyDescriptor deltaframe_descriptor;
1165 deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0];
1166 deltaframe_descriptor.frame_number = 4;
1167 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1168 .WillOnce([&](video_coding::EncodedFrame* frame) {
1169 EXPECT_EQ(frame->id.picture_id & 0xFFFF, 4);
1170 });
1171 InjectPacketWith(stream_structure2, deltaframe_descriptor);
1172 }
1173
1174 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
1175 using RtpVideoStreamReceiverDeathTest = RtpVideoStreamReceiverTest;
TEST_F(RtpVideoStreamReceiverDeathTest,RepeatedSecondarySinkDisallowed)1176 TEST_F(RtpVideoStreamReceiverDeathTest, RepeatedSecondarySinkDisallowed) {
1177 MockRtpPacketSink secondary_sink;
1178
1179 rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink);
1180 EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink),
1181 "");
1182
1183 // Test tear-down.
1184 rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink);
1185 }
1186 #endif
1187
TEST_F(RtpVideoStreamReceiverTest,TransformFrame)1188 TEST_F(RtpVideoStreamReceiverTest, TransformFrame) {
1189 rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
1190 new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>();
1191 EXPECT_CALL(*mock_frame_transformer,
1192 RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc));
1193 auto receiver = std::make_unique<RtpVideoStreamReceiver>(
1194 Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
1195 rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(),
1196 &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr,
1197 mock_frame_transformer);
1198 VideoCodec video_codec;
1199 video_codec.codecType = kVideoCodecGeneric;
1200 receiver->AddReceiveCodec(kPayloadType, video_codec, {},
1201 /*raw_payload=*/false);
1202
1203 RtpPacketReceived rtp_packet;
1204 rtp_packet.SetPayloadType(kPayloadType);
1205 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
1206 rtp_packet.SetSequenceNumber(1);
1207 RTPVideoHeader video_header =
1208 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
1209 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
1210 data.size());
1211 EXPECT_CALL(*mock_frame_transformer, Transform(_));
1212 receiver->OnReceivedPayloadData(data, rtp_packet, video_header);
1213
1214 EXPECT_CALL(*mock_frame_transformer,
1215 UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc));
1216 receiver = nullptr;
1217 }
1218
1219 // Test default behavior and when playout delay is overridden by field trial.
1220 const VideoPlayoutDelay kTransmittedPlayoutDelay = {100, 200};
1221 const VideoPlayoutDelay kForcedPlayoutDelay = {70, 90};
1222 struct PlayoutDelayOptions {
1223 std::string field_trial;
1224 VideoPlayoutDelay expected_delay;
1225 };
1226 const PlayoutDelayOptions kDefaultBehavior = {
1227 /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay};
1228 const PlayoutDelayOptions kOverridePlayoutDelay = {
1229 /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/",
1230 /*expected_delay=*/kForcedPlayoutDelay};
1231
1232 class RtpVideoStreamReceiverTestPlayoutDelay
1233 : public RtpVideoStreamReceiverTest,
1234 public ::testing::WithParamInterface<PlayoutDelayOptions> {
1235 protected:
RtpVideoStreamReceiverTestPlayoutDelay()1236 RtpVideoStreamReceiverTestPlayoutDelay()
1237 : RtpVideoStreamReceiverTest(GetParam().field_trial) {}
1238 };
1239
1240 INSTANTIATE_TEST_SUITE_P(PlayoutDelay,
1241 RtpVideoStreamReceiverTestPlayoutDelay,
1242 Values(kDefaultBehavior, kOverridePlayoutDelay));
1243
TEST_P(RtpVideoStreamReceiverTestPlayoutDelay,PlayoutDelay)1244 TEST_P(RtpVideoStreamReceiverTestPlayoutDelay, PlayoutDelay) {
1245 rtc::CopyOnWriteBuffer payload_data({1, 2, 3, 4});
1246 RtpHeaderExtensionMap extension_map;
1247 extension_map.Register<PlayoutDelayLimits>(1);
1248 RtpPacketToSend packet_to_send(&extension_map);
1249 packet_to_send.SetPayloadType(kPayloadType);
1250 packet_to_send.SetSequenceNumber(1);
1251
1252 // Set playout delay on outgoing packet.
1253 EXPECT_TRUE(packet_to_send.SetExtension<PlayoutDelayLimits>(
1254 kTransmittedPlayoutDelay));
1255 uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size());
1256 memcpy(payload, payload_data.data(), payload_data.size());
1257
1258 RtpPacketReceived received_packet(&extension_map);
1259 received_packet.Parse(packet_to_send.data(), packet_to_send.size());
1260
1261 RTPVideoHeader video_header =
1262 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
1263 mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(),
1264 payload_data.size());
1265 // Expect the playout delay of encoded frame to be the same as the transmitted
1266 // playout delay unless it was overridden by a field trial.
1267 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
1268 .WillOnce(Invoke([expected_playout_delay = GetParam().expected_delay](
1269 video_coding::EncodedFrame* frame) {
1270 EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay);
1271 }));
1272 rtp_video_stream_receiver_->OnReceivedPayloadData(
1273 received_packet.PayloadBuffer(), received_packet, video_header);
1274 }
1275
1276 } // namespace webrtc
1277