1 /*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/rtp_video_stream_receiver2.h"
12
13 #include <memory>
14 #include <utility>
15
16 #include "api/video/video_codec_type.h"
17 #include "api/video/video_frame_type.h"
18 #include "common_video/h264/h264_common.h"
19 #include "media/base/media_constants.h"
20 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
21 #include "modules/rtp_rtcp/source/rtp_format.h"
22 #include "modules/rtp_rtcp/source/rtp_format_vp9.h"
23 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
24 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
25 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
26 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
27 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
28 #include "modules/utility/include/process_thread.h"
29 #include "modules/video_coding/frame_object.h"
30 #include "modules/video_coding/include/video_coding_defines.h"
31 #include "modules/video_coding/rtp_frame_reference_finder.h"
32 #include "rtc_base/byte_buffer.h"
33 #include "rtc_base/logging.h"
34 #include "system_wrappers/include/clock.h"
35 #include "system_wrappers/include/field_trial.h"
36 #include "test/field_trial.h"
37 #include "test/gmock.h"
38 #include "test/gtest.h"
39 #include "test/mock_frame_transformer.h"
40 #include "test/time_controller/simulated_task_queue.h"
41
42 using ::testing::_;
43 using ::testing::ElementsAre;
44 using ::testing::Invoke;
45 using ::testing::SizeIs;
46 using ::testing::Values;
47
48 namespace webrtc {
49
50 namespace {
51
52 const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01};
53
GetAbsoluteCaptureTimestamps(const EncodedFrame * frame)54 std::vector<uint64_t> GetAbsoluteCaptureTimestamps(const EncodedFrame* frame) {
55 std::vector<uint64_t> result;
56 for (const auto& packet_info : frame->PacketInfos()) {
57 if (packet_info.absolute_capture_time()) {
58 result.push_back(
59 packet_info.absolute_capture_time()->absolute_capture_timestamp);
60 }
61 }
62 return result;
63 }
64
GetGenericVideoHeader(VideoFrameType frame_type)65 RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) {
66 RTPVideoHeader video_header;
67 video_header.is_first_packet_in_frame = true;
68 video_header.is_last_packet_in_frame = true;
69 video_header.codec = kVideoCodecGeneric;
70 video_header.frame_type = frame_type;
71 return video_header;
72 }
73
74 class MockTransport : public Transport {
75 public:
76 MOCK_METHOD(bool,
77 SendRtp,
78 (const uint8_t*, size_t length, const PacketOptions& options),
79 (override));
80 MOCK_METHOD(bool, SendRtcp, (const uint8_t*, size_t length), (override));
81 };
82
83 class MockNackSender : public NackSender {
84 public:
85 MOCK_METHOD(void,
86 SendNack,
87 (const std::vector<uint16_t>& sequence_numbers,
88 bool buffering_allowed),
89 (override));
90 };
91
92 class MockKeyFrameRequestSender : public KeyFrameRequestSender {
93 public:
94 MOCK_METHOD(void, RequestKeyFrame, (), (override));
95 };
96
97 class MockOnCompleteFrameCallback : public OnCompleteFrameCallback {
98 public:
99 MOCK_METHOD(void, DoOnCompleteFrame, (EncodedFrame*), ());
100 MOCK_METHOD(void, DoOnCompleteFrameFailNullptr, (EncodedFrame*), ());
101 MOCK_METHOD(void, DoOnCompleteFrameFailLength, (EncodedFrame*), ());
102 MOCK_METHOD(void, DoOnCompleteFrameFailBitstream, (EncodedFrame*), ());
OnCompleteFrame(std::unique_ptr<EncodedFrame> frame)103 void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override {
104 if (!frame) {
105 DoOnCompleteFrameFailNullptr(nullptr);
106 return;
107 }
108 EXPECT_EQ(buffer_.Length(), frame->size());
109 if (buffer_.Length() != frame->size()) {
110 DoOnCompleteFrameFailLength(frame.get());
111 return;
112 }
113 if (frame->size() != buffer_.Length() ||
114 memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) {
115 DoOnCompleteFrameFailBitstream(frame.get());
116 return;
117 }
118 DoOnCompleteFrame(frame.get());
119 }
120
ClearExpectedBitstream()121 void ClearExpectedBitstream() { buffer_.Clear(); }
122
AppendExpectedBitstream(const uint8_t data[],size_t size_in_bytes)123 void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) {
124 // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*.
125 buffer_.WriteBytes(reinterpret_cast<const char*>(data), size_in_bytes);
126 }
127 rtc::ByteBufferWriter buffer_;
128 };
129
130 class MockRtpPacketSink : public RtpPacketSinkInterface {
131 public:
132 MOCK_METHOD(void, OnRtpPacket, (const RtpPacketReceived&), (override));
133 };
134
135 constexpr uint32_t kSsrc = 111;
136 constexpr int kPayloadType = 100;
137 constexpr int kRedPayloadType = 125;
138
CreateRtpPacketReceived()139 std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived() {
140 constexpr uint16_t kSequenceNumber = 222;
141 auto packet = std::make_unique<RtpPacketReceived>();
142 packet->SetSsrc(kSsrc);
143 packet->SetSequenceNumber(kSequenceNumber);
144 packet->SetPayloadType(kPayloadType);
145 return packet;
146 }
147
148 MATCHER_P(SamePacketAs, other, "") {
149 return arg.Ssrc() == other.Ssrc() &&
150 arg.SequenceNumber() == other.SequenceNumber();
151 }
152
153 } // namespace
154
155 class RtpVideoStreamReceiver2Test : public ::testing::Test,
156 public RtpPacketSinkInterface {
157 public:
RtpVideoStreamReceiver2Test()158 RtpVideoStreamReceiver2Test() : RtpVideoStreamReceiver2Test("") {}
RtpVideoStreamReceiver2Test(std::string field_trials)159 explicit RtpVideoStreamReceiver2Test(std::string field_trials)
160 : override_field_trials_(field_trials),
161 config_(CreateConfig()),
162 process_thread_(ProcessThread::Create("TestThread")) {
163 rtp_receive_statistics_ =
164 ReceiveStatistics::Create(Clock::GetRealTimeClock());
165 rtp_video_stream_receiver_ = std::make_unique<RtpVideoStreamReceiver2>(
166 TaskQueueBase::Current(), Clock::GetRealTimeClock(), &mock_transport_,
167 nullptr, nullptr, &config_, rtp_receive_statistics_.get(), nullptr,
168 nullptr, process_thread_.get(), &mock_nack_sender_,
169 &mock_key_frame_request_sender_, &mock_on_complete_frame_callback_,
170 nullptr, nullptr);
171 VideoCodec codec;
172 codec.codecType = kVideoCodecGeneric;
173 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, {},
174 /*raw_payload=*/false);
175 }
176
GetDefaultH264VideoHeader()177 RTPVideoHeader GetDefaultH264VideoHeader() {
178 RTPVideoHeader video_header;
179 video_header.codec = kVideoCodecH264;
180 video_header.video_type_header.emplace<RTPVideoHeaderH264>();
181 return video_header;
182 }
183
184 // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate
185 // code.
AddSps(RTPVideoHeader * video_header,uint8_t sps_id,rtc::CopyOnWriteBuffer * data)186 void AddSps(RTPVideoHeader* video_header,
187 uint8_t sps_id,
188 rtc::CopyOnWriteBuffer* data) {
189 NaluInfo info;
190 info.type = H264::NaluType::kSps;
191 info.sps_id = sps_id;
192 info.pps_id = -1;
193 data->AppendData<uint8_t, 2>({H264::NaluType::kSps, sps_id});
194 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
195 h264.nalus[h264.nalus_length++] = info;
196 }
197
AddPps(RTPVideoHeader * video_header,uint8_t sps_id,uint8_t pps_id,rtc::CopyOnWriteBuffer * data)198 void AddPps(RTPVideoHeader* video_header,
199 uint8_t sps_id,
200 uint8_t pps_id,
201 rtc::CopyOnWriteBuffer* data) {
202 NaluInfo info;
203 info.type = H264::NaluType::kPps;
204 info.sps_id = sps_id;
205 info.pps_id = pps_id;
206 data->AppendData<uint8_t, 2>({H264::NaluType::kPps, pps_id});
207 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
208 h264.nalus[h264.nalus_length++] = info;
209 }
210
AddIdr(RTPVideoHeader * video_header,int pps_id)211 void AddIdr(RTPVideoHeader* video_header, int pps_id) {
212 NaluInfo info;
213 info.type = H264::NaluType::kIdr;
214 info.sps_id = -1;
215 info.pps_id = pps_id;
216 auto& h264 = absl::get<RTPVideoHeaderH264>(video_header->video_type_header);
217 h264.nalus[h264.nalus_length++] = info;
218 }
219
OnRtpPacket(const RtpPacketReceived & packet)220 void OnRtpPacket(const RtpPacketReceived& packet) override {
221 if (test_packet_sink_)
222 test_packet_sink_->OnRtpPacket(packet);
223 }
224
225 protected:
CreateConfig()226 VideoReceiveStream::Config CreateConfig() {
227 VideoReceiveStream::Config config(nullptr);
228 config.rtp.remote_ssrc = 1111;
229 config.rtp.local_ssrc = 2222;
230 config.rtp.red_payload_type = kRedPayloadType;
231 config.rtp.packet_sink_ = this;
232 return config;
233 }
234
235 TokenTaskQueue task_queue_;
236 TokenTaskQueue::CurrentTaskQueueSetter task_queue_setter_{&task_queue_};
237
238 const webrtc::test::ScopedFieldTrials override_field_trials_;
239 VideoReceiveStream::Config config_;
240 MockNackSender mock_nack_sender_;
241 MockKeyFrameRequestSender mock_key_frame_request_sender_;
242 MockTransport mock_transport_;
243 MockOnCompleteFrameCallback mock_on_complete_frame_callback_;
244 std::unique_ptr<ProcessThread> process_thread_;
245 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
246 std::unique_ptr<RtpVideoStreamReceiver2> rtp_video_stream_receiver_;
247 RtpPacketSinkInterface* test_packet_sink_ = nullptr;
248 };
249
TEST_F(RtpVideoStreamReceiver2Test,CacheColorSpaceFromLastPacketOfKeyframe)250 TEST_F(RtpVideoStreamReceiver2Test, CacheColorSpaceFromLastPacketOfKeyframe) {
251 // Test that color space is cached from the last packet of a key frame and
252 // that it's not reset by padding packets without color space.
253 constexpr int kVp9PayloadType = 99;
254 const ColorSpace kColorSpace(
255 ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12,
256 ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull);
257 const std::vector<uint8_t> kKeyFramePayload = {0, 1, 2, 3, 4, 5,
258 6, 7, 8, 9, 10};
259 const std::vector<uint8_t> kDeltaFramePayload = {0, 1, 2, 3, 4};
260
261 // Anonymous helper class that generates received packets.
262 class {
263 public:
264 void SetPayload(const std::vector<uint8_t>& payload,
265 VideoFrameType video_frame_type) {
266 video_frame_type_ = video_frame_type;
267 RtpPacketizer::PayloadSizeLimits pay_load_size_limits;
268 // Reduce max payload length to make sure the key frame generates two
269 // packets.
270 pay_load_size_limits.max_payload_len = 8;
271 RTPVideoHeaderVP9 rtp_video_header_vp9;
272 rtp_video_header_vp9.InitRTPVideoHeaderVP9();
273 rtp_video_header_vp9.inter_pic_predicted =
274 (video_frame_type == VideoFrameType::kVideoFrameDelta);
275 rtp_packetizer_ = std::make_unique<RtpPacketizerVp9>(
276 payload, pay_load_size_limits, rtp_video_header_vp9);
277 }
278
279 size_t NumPackets() { return rtp_packetizer_->NumPackets(); }
280 void SetColorSpace(const ColorSpace& color_space) {
281 color_space_ = color_space;
282 }
283
284 RtpPacketReceived NextPacket() {
285 RtpHeaderExtensionMap extension_map;
286 extension_map.Register<ColorSpaceExtension>(1);
287 RtpPacketToSend packet_to_send(&extension_map);
288 packet_to_send.SetSequenceNumber(sequence_number_++);
289 packet_to_send.SetSsrc(kSsrc);
290 packet_to_send.SetPayloadType(kVp9PayloadType);
291 bool include_color_space =
292 (rtp_packetizer_->NumPackets() == 1u &&
293 video_frame_type_ == VideoFrameType::kVideoFrameKey);
294 if (include_color_space) {
295 EXPECT_TRUE(
296 packet_to_send.SetExtension<ColorSpaceExtension>(color_space_));
297 }
298 rtp_packetizer_->NextPacket(&packet_to_send);
299
300 RtpPacketReceived received_packet(&extension_map);
301 received_packet.Parse(packet_to_send.data(), packet_to_send.size());
302 return received_packet;
303 }
304
305 private:
306 uint16_t sequence_number_ = 0;
307 VideoFrameType video_frame_type_;
308 ColorSpace color_space_;
309 std::unique_ptr<RtpPacketizer> rtp_packetizer_;
310 } received_packet_generator;
311 received_packet_generator.SetColorSpace(kColorSpace);
312
313 // Prepare the receiver for VP9.
314 VideoCodec codec;
315 codec.codecType = kVideoCodecVP9;
316 std::map<std::string, std::string> codec_params;
317 rtp_video_stream_receiver_->AddReceiveCodec(kVp9PayloadType, codec,
318 codec_params,
319 /*raw_payload=*/false);
320
321 // Generate key frame packets.
322 received_packet_generator.SetPayload(kKeyFramePayload,
323 VideoFrameType::kVideoFrameKey);
324 EXPECT_EQ(received_packet_generator.NumPackets(), 2u);
325 RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket();
326 RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket();
327
328 // Generate delta frame packet.
329 received_packet_generator.SetPayload(kDeltaFramePayload,
330 VideoFrameType::kVideoFrameDelta);
331 EXPECT_EQ(received_packet_generator.NumPackets(), 1u);
332 RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket();
333
334 rtp_video_stream_receiver_->StartReceive();
335 mock_on_complete_frame_callback_.AppendExpectedBitstream(
336 kKeyFramePayload.data(), kKeyFramePayload.size());
337
338 // Send the key frame and expect a callback with color space information.
339 EXPECT_FALSE(key_frame_packet1.GetExtension<ColorSpaceExtension>());
340 EXPECT_TRUE(key_frame_packet2.GetExtension<ColorSpaceExtension>());
341 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
342 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
343 .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) {
344 ASSERT_TRUE(frame->EncodedImage().ColorSpace());
345 EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
346 }));
347 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2);
348 // Resend the first key frame packet to simulate padding for example.
349 rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1);
350
351 mock_on_complete_frame_callback_.ClearExpectedBitstream();
352 mock_on_complete_frame_callback_.AppendExpectedBitstream(
353 kDeltaFramePayload.data(), kDeltaFramePayload.size());
354
355 // Expect delta frame to have color space set even though color space not
356 // included in the RTP packet.
357 EXPECT_FALSE(delta_frame_packet.GetExtension<ColorSpaceExtension>());
358 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
359 .WillOnce(Invoke([kColorSpace](EncodedFrame* frame) {
360 ASSERT_TRUE(frame->EncodedImage().ColorSpace());
361 EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace);
362 }));
363 rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet);
364 }
365
TEST_F(RtpVideoStreamReceiver2Test,GenericKeyFrame)366 TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrame) {
367 RtpPacketReceived rtp_packet;
368 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
369 rtp_packet.SetPayloadType(kPayloadType);
370 rtp_packet.SetSequenceNumber(1);
371 RTPVideoHeader video_header =
372 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
373 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
374 data.size());
375 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
376 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
377 video_header);
378 }
379
TEST_F(RtpVideoStreamReceiver2Test,PacketInfoIsPropagatedIntoVideoFrames)380 TEST_F(RtpVideoStreamReceiver2Test, PacketInfoIsPropagatedIntoVideoFrames) {
381 constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
382 constexpr int kId0 = 1;
383
384 RtpHeaderExtensionMap extension_map;
385 extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
386 RtpPacketReceived rtp_packet(&extension_map);
387 rtp_packet.SetPayloadType(kPayloadType);
388 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
389 rtp_packet.SetSequenceNumber(1);
390 rtp_packet.SetTimestamp(1);
391 rtp_packet.SetSsrc(kSsrc);
392 rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
393 AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
394 /*estimated_capture_clock_offset=*/absl::nullopt});
395
396 RTPVideoHeader video_header =
397 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
398 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
399 data.size());
400 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
401 .WillOnce(Invoke([kAbsoluteCaptureTimestamp](EncodedFrame* frame) {
402 EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame),
403 ElementsAre(kAbsoluteCaptureTimestamp));
404 }));
405 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
406 video_header);
407 }
408
TEST_F(RtpVideoStreamReceiver2Test,MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue)409 TEST_F(RtpVideoStreamReceiver2Test,
410 MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) {
411 constexpr uint64_t kAbsoluteCaptureTimestamp = 12;
412 constexpr int kId0 = 1;
413
414 RtpHeaderExtensionMap extension_map;
415 extension_map.Register<AbsoluteCaptureTimeExtension>(kId0);
416 RtpPacketReceived rtp_packet(&extension_map);
417 rtp_packet.SetPayloadType(kPayloadType);
418
419 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
420 uint16_t sequence_number = 1;
421 uint32_t rtp_timestamp = 1;
422 rtp_packet.SetSequenceNumber(sequence_number);
423 rtp_packet.SetTimestamp(rtp_timestamp);
424 rtp_packet.SetSsrc(kSsrc);
425 rtp_packet.SetExtension<AbsoluteCaptureTimeExtension>(
426 AbsoluteCaptureTime{kAbsoluteCaptureTimestamp,
427 /*estimated_capture_clock_offset=*/absl::nullopt});
428
429 RTPVideoHeader video_header =
430 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
431 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
432 data.size());
433 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
434 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
435 video_header);
436
437 // Rtp packet without absolute capture time.
438 rtp_packet = RtpPacketReceived(&extension_map);
439 rtp_packet.SetPayloadType(kPayloadType);
440 rtp_packet.SetSequenceNumber(++sequence_number);
441 rtp_packet.SetTimestamp(++rtp_timestamp);
442 rtp_packet.SetSsrc(kSsrc);
443
444 // There is no absolute capture time in the second packet.
445 // Expect rtp video stream receiver to extrapolate it for the resulting video
446 // frame using absolute capture time from the previous packet.
447 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
448 .WillOnce(Invoke([](EncodedFrame* frame) {
449 EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1));
450 }));
451 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
452 video_header);
453 }
454
TEST_F(RtpVideoStreamReceiver2Test,NoInfiniteRecursionOnEncapsulatedRedPacket)455 TEST_F(RtpVideoStreamReceiver2Test,
456 NoInfiniteRecursionOnEncapsulatedRedPacket) {
457 const std::vector<uint8_t> data({
458 0x80, // RTP version.
459 kRedPayloadType, // Payload type.
460 0, 0, 0, 0, 0, 0, // Don't care.
461 0, 0, 0x4, 0x57, // SSRC
462 kRedPayloadType, // RED header.
463 0, 0, 0, 0, 0 // Don't care.
464 });
465 RtpPacketReceived packet;
466 EXPECT_TRUE(packet.Parse(data.data(), data.size()));
467 rtp_video_stream_receiver_->StartReceive();
468 rtp_video_stream_receiver_->OnRtpPacket(packet);
469 }
470
TEST_F(RtpVideoStreamReceiver2Test,DropsPacketWithRedPayloadTypeAndEmptyPayload)471 TEST_F(RtpVideoStreamReceiver2Test,
472 DropsPacketWithRedPayloadTypeAndEmptyPayload) {
473 const uint8_t kRedPayloadType = 125;
474 config_.rtp.red_payload_type = kRedPayloadType;
475 SetUp(); // re-create rtp_video_stream_receiver with red payload type.
476 // clang-format off
477 const uint8_t data[] = {
478 0x80, // RTP version.
479 kRedPayloadType, // Payload type.
480 0, 0, 0, 0, 0, 0, // Don't care.
481 0, 0, 0x4, 0x57, // SSRC
482 // Empty rtp payload.
483 };
484 // clang-format on
485 RtpPacketReceived packet;
486 // Manually convert to CopyOnWriteBuffer to be sure capacity == size
487 // and asan bot can catch read buffer overflow.
488 EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data)));
489 rtp_video_stream_receiver_->StartReceive();
490 rtp_video_stream_receiver_->OnRtpPacket(packet);
491 // Expect asan doesn't find anything.
492 }
493
TEST_F(RtpVideoStreamReceiver2Test,GenericKeyFrameBitstreamError)494 TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrameBitstreamError) {
495 RtpPacketReceived rtp_packet;
496 rtp_packet.SetPayloadType(kPayloadType);
497 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
498 rtp_packet.SetSequenceNumber(1);
499 RTPVideoHeader video_header =
500 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
501 constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff};
502 mock_on_complete_frame_callback_.AppendExpectedBitstream(
503 expected_bitsteam, sizeof(expected_bitsteam));
504 EXPECT_CALL(mock_on_complete_frame_callback_,
505 DoOnCompleteFrameFailBitstream(_));
506 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
507 video_header);
508 }
509
510 class RtpVideoStreamReceiver2TestH264
511 : public RtpVideoStreamReceiver2Test,
512 public ::testing::WithParamInterface<std::string> {
513 protected:
RtpVideoStreamReceiver2TestH264()514 RtpVideoStreamReceiver2TestH264() : RtpVideoStreamReceiver2Test(GetParam()) {}
515 };
516
517 INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe,
518 RtpVideoStreamReceiver2TestH264,
519 Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/"));
520
TEST_P(RtpVideoStreamReceiver2TestH264,InBandSpsPps)521 TEST_P(RtpVideoStreamReceiver2TestH264, InBandSpsPps) {
522 rtc::CopyOnWriteBuffer sps_data;
523 RtpPacketReceived rtp_packet;
524 RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
525 AddSps(&sps_video_header, 0, &sps_data);
526 rtp_packet.SetSequenceNumber(0);
527 rtp_packet.SetPayloadType(kPayloadType);
528 sps_video_header.is_first_packet_in_frame = true;
529 sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
530 mock_on_complete_frame_callback_.AppendExpectedBitstream(
531 kH264StartCode, sizeof(kH264StartCode));
532 mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
533 sps_data.size());
534 rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
535 sps_video_header);
536
537 rtc::CopyOnWriteBuffer pps_data;
538 RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
539 AddPps(&pps_video_header, 0, 1, &pps_data);
540 rtp_packet.SetSequenceNumber(1);
541 pps_video_header.is_first_packet_in_frame = true;
542 pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
543 mock_on_complete_frame_callback_.AppendExpectedBitstream(
544 kH264StartCode, sizeof(kH264StartCode));
545 mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
546 pps_data.size());
547 rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
548 pps_video_header);
549
550 rtc::CopyOnWriteBuffer idr_data;
551 RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
552 AddIdr(&idr_video_header, 1);
553 rtp_packet.SetSequenceNumber(2);
554 idr_video_header.is_first_packet_in_frame = true;
555 idr_video_header.is_last_packet_in_frame = true;
556 idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
557 const uint8_t idr[] = {0x65, 1, 2, 3};
558 idr_data.AppendData(idr);
559 mock_on_complete_frame_callback_.AppendExpectedBitstream(
560 kH264StartCode, sizeof(kH264StartCode));
561 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
562 idr_data.size());
563 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
564 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
565 idr_video_header);
566 }
567
TEST_P(RtpVideoStreamReceiver2TestH264,OutOfBandFmtpSpsPps)568 TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) {
569 constexpr int kPayloadType = 99;
570 VideoCodec codec;
571 std::map<std::string, std::string> codec_params;
572 // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2
573 // .
574 codec_params.insert(
575 {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="});
576 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, codec_params,
577 /*raw_payload=*/false);
578 const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96,
579 0x53, 0x05, 0x89, 0x88};
580 mock_on_complete_frame_callback_.AppendExpectedBitstream(
581 kH264StartCode, sizeof(kH264StartCode));
582 mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps,
583 sizeof(binary_sps));
584 const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88};
585 mock_on_complete_frame_callback_.AppendExpectedBitstream(
586 kH264StartCode, sizeof(kH264StartCode));
587 mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps,
588 sizeof(binary_pps));
589
590 RtpPacketReceived rtp_packet;
591 RTPVideoHeader video_header = GetDefaultH264VideoHeader();
592 AddIdr(&video_header, 0);
593 rtp_packet.SetPayloadType(kPayloadType);
594 rtp_packet.SetSequenceNumber(2);
595 video_header.is_first_packet_in_frame = true;
596 video_header.is_last_packet_in_frame = true;
597 video_header.codec = kVideoCodecH264;
598 video_header.frame_type = VideoFrameType::kVideoFrameKey;
599 rtc::CopyOnWriteBuffer data({1, 2, 3});
600 mock_on_complete_frame_callback_.AppendExpectedBitstream(
601 kH264StartCode, sizeof(kH264StartCode));
602 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
603 data.size());
604 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
605 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
606 video_header);
607 }
608
TEST_P(RtpVideoStreamReceiver2TestH264,ForceSpsPpsIdrIsKeyframe)609 TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) {
610 constexpr int kPayloadType = 99;
611 VideoCodec codec;
612 std::map<std::string, std::string> codec_params;
613 if (GetParam() ==
614 "") { // Forcing can be done either with field trial or codec_params.
615 codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""});
616 }
617 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, codec_params,
618 /*raw_payload=*/false);
619 rtc::CopyOnWriteBuffer sps_data;
620 RtpPacketReceived rtp_packet;
621 RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader();
622 AddSps(&sps_video_header, 0, &sps_data);
623 rtp_packet.SetSequenceNumber(0);
624 rtp_packet.SetPayloadType(kPayloadType);
625 sps_video_header.is_first_packet_in_frame = true;
626 sps_video_header.frame_type = VideoFrameType::kEmptyFrame;
627 mock_on_complete_frame_callback_.AppendExpectedBitstream(
628 kH264StartCode, sizeof(kH264StartCode));
629 mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(),
630 sps_data.size());
631 rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet,
632 sps_video_header);
633
634 rtc::CopyOnWriteBuffer pps_data;
635 RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader();
636 AddPps(&pps_video_header, 0, 1, &pps_data);
637 rtp_packet.SetSequenceNumber(1);
638 pps_video_header.is_first_packet_in_frame = true;
639 pps_video_header.frame_type = VideoFrameType::kEmptyFrame;
640 mock_on_complete_frame_callback_.AppendExpectedBitstream(
641 kH264StartCode, sizeof(kH264StartCode));
642 mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(),
643 pps_data.size());
644 rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet,
645 pps_video_header);
646
647 rtc::CopyOnWriteBuffer idr_data;
648 RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader();
649 AddIdr(&idr_video_header, 1);
650 rtp_packet.SetSequenceNumber(2);
651 idr_video_header.is_first_packet_in_frame = true;
652 idr_video_header.is_last_packet_in_frame = true;
653 idr_video_header.frame_type = VideoFrameType::kVideoFrameKey;
654 const uint8_t idr[] = {0x65, 1, 2, 3};
655 idr_data.AppendData(idr);
656 mock_on_complete_frame_callback_.AppendExpectedBitstream(
657 kH264StartCode, sizeof(kH264StartCode));
658 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
659 idr_data.size());
660 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
661 .WillOnce(
662 [&](EncodedFrame* frame) { EXPECT_TRUE(frame->is_keyframe()); });
663 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
664 idr_video_header);
665 mock_on_complete_frame_callback_.ClearExpectedBitstream();
666 mock_on_complete_frame_callback_.AppendExpectedBitstream(
667 kH264StartCode, sizeof(kH264StartCode));
668 mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(),
669 idr_data.size());
670 rtp_packet.SetSequenceNumber(3);
671 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
672 .WillOnce(
673 [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); });
674 rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet,
675 idr_video_header);
676 }
677
TEST_F(RtpVideoStreamReceiver2Test,PaddingInMediaStream)678 TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) {
679 RtpPacketReceived rtp_packet;
680 RTPVideoHeader video_header = GetDefaultH264VideoHeader();
681 rtc::CopyOnWriteBuffer data({1, 2, 3});
682 rtp_packet.SetPayloadType(kPayloadType);
683 rtp_packet.SetSequenceNumber(2);
684 video_header.is_first_packet_in_frame = true;
685 video_header.is_last_packet_in_frame = true;
686 video_header.codec = kVideoCodecGeneric;
687 video_header.frame_type = VideoFrameType::kVideoFrameKey;
688 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
689 data.size());
690
691 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
692 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
693 video_header);
694
695 rtp_packet.SetSequenceNumber(3);
696 rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
697 video_header);
698
699 rtp_packet.SetSequenceNumber(4);
700 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
701 video_header.frame_type = VideoFrameType::kVideoFrameDelta;
702 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
703 video_header);
704
705 rtp_packet.SetSequenceNumber(6);
706 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
707 video_header);
708
709 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_));
710 rtp_packet.SetSequenceNumber(5);
711 rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet,
712 video_header);
713 }
714
TEST_F(RtpVideoStreamReceiver2Test,RequestKeyframeIfFirstFrameIsDelta)715 TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeIfFirstFrameIsDelta) {
716 RtpPacketReceived rtp_packet;
717 rtp_packet.SetPayloadType(kPayloadType);
718 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
719 rtp_packet.SetSequenceNumber(1);
720 RTPVideoHeader video_header =
721 GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
722 EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
723 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
724 video_header);
725 }
726
TEST_F(RtpVideoStreamReceiver2Test,RequestKeyframeWhenPacketBufferGetsFull)727 TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeWhenPacketBufferGetsFull) {
728 constexpr int kPacketBufferMaxSize = 2048;
729
730 RtpPacketReceived rtp_packet;
731 rtp_packet.SetPayloadType(kPayloadType);
732 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
733 RTPVideoHeader video_header =
734 GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta);
735 // Incomplete frames so that the packet buffer is filling up.
736 video_header.is_last_packet_in_frame = false;
737 uint16_t start_sequence_number = 1234;
738 rtp_packet.SetSequenceNumber(start_sequence_number);
739 while (rtp_packet.SequenceNumber() - start_sequence_number <
740 kPacketBufferMaxSize) {
741 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
742 video_header);
743 rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2);
744 }
745
746 EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame());
747 rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet,
748 video_header);
749 }
750
TEST_F(RtpVideoStreamReceiver2Test,SinkGetsRtpNotifications)751 TEST_F(RtpVideoStreamReceiver2Test, SinkGetsRtpNotifications) {
752 rtp_video_stream_receiver_->StartReceive();
753
754 MockRtpPacketSink test_sink;
755 test_packet_sink_ = &test_sink;
756
757 auto rtp_packet = CreateRtpPacketReceived();
758 EXPECT_CALL(test_sink, OnRtpPacket(SamePacketAs(*rtp_packet)));
759
760 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
761
762 // Test tear-down.
763 rtp_video_stream_receiver_->StopReceive();
764 test_packet_sink_ = nullptr;
765 }
766
TEST_F(RtpVideoStreamReceiver2Test,NonStartedStreamGetsNoRtpCallbacks)767 TEST_F(RtpVideoStreamReceiver2Test, NonStartedStreamGetsNoRtpCallbacks) {
768 // Explicitly showing that the stream is not in the |started| state,
769 // regardless of whether streams start out |started| or |stopped|.
770 rtp_video_stream_receiver_->StopReceive();
771
772 MockRtpPacketSink test_sink;
773 test_packet_sink_ = &test_sink;
774
775 auto rtp_packet = CreateRtpPacketReceived();
776 EXPECT_CALL(test_sink, OnRtpPacket(_)).Times(0);
777
778 rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet);
779
780 test_packet_sink_ = nullptr;
781 }
782
TEST_F(RtpVideoStreamReceiver2Test,ParseGenericDescriptorOnePacket)783 TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorOnePacket) {
784 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
785 const int kSpatialIndex = 1;
786
787 rtp_video_stream_receiver_->StartReceive();
788
789 RtpHeaderExtensionMap extension_map;
790 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
791 RtpPacketReceived rtp_packet(&extension_map);
792 rtp_packet.SetPayloadType(kPayloadType);
793
794 RtpGenericFrameDescriptor generic_descriptor;
795 generic_descriptor.SetFirstPacketInSubFrame(true);
796 generic_descriptor.SetLastPacketInSubFrame(true);
797 generic_descriptor.SetFrameId(100);
798 generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
799 generic_descriptor.AddFrameDependencyDiff(90);
800 generic_descriptor.AddFrameDependencyDiff(80);
801 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
802 generic_descriptor));
803
804 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
805 memcpy(payload, data.data(), data.size());
806 // The first byte is the header, so we ignore the first byte of |data|.
807 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
808 data.size() - 1);
809
810 rtp_packet.SetMarker(true);
811 rtp_packet.SetPayloadType(kPayloadType);
812 rtp_packet.SetSequenceNumber(1);
813
814 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
815 .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) {
816 EXPECT_EQ(frame->num_references, 2U);
817 EXPECT_EQ(frame->references[0], frame->Id() - 90);
818 EXPECT_EQ(frame->references[1], frame->Id() - 80);
819 EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex);
820 EXPECT_THAT(frame->PacketInfos(), SizeIs(1));
821 }));
822
823 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
824 }
825
TEST_F(RtpVideoStreamReceiver2Test,ParseGenericDescriptorTwoPackets)826 TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorTwoPackets) {
827 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
828 const int kSpatialIndex = 1;
829
830 rtp_video_stream_receiver_->StartReceive();
831
832 RtpHeaderExtensionMap extension_map;
833 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
834 RtpPacketReceived first_packet(&extension_map);
835
836 RtpGenericFrameDescriptor first_packet_descriptor;
837 first_packet_descriptor.SetFirstPacketInSubFrame(true);
838 first_packet_descriptor.SetLastPacketInSubFrame(false);
839 first_packet_descriptor.SetFrameId(100);
840 first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex);
841 first_packet_descriptor.SetResolution(480, 360);
842 ASSERT_TRUE(first_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
843 first_packet_descriptor));
844
845 uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size());
846 memcpy(first_packet_payload, data.data(), data.size());
847 // The first byte is the header, so we ignore the first byte of |data|.
848 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
849 data.size() - 1);
850
851 first_packet.SetPayloadType(kPayloadType);
852 first_packet.SetSequenceNumber(1);
853 rtp_video_stream_receiver_->OnRtpPacket(first_packet);
854
855 RtpPacketReceived second_packet(&extension_map);
856 RtpGenericFrameDescriptor second_packet_descriptor;
857 second_packet_descriptor.SetFirstPacketInSubFrame(false);
858 second_packet_descriptor.SetLastPacketInSubFrame(true);
859 ASSERT_TRUE(second_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
860 second_packet_descriptor));
861
862 second_packet.SetMarker(true);
863 second_packet.SetPayloadType(kPayloadType);
864 second_packet.SetSequenceNumber(2);
865
866 uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size());
867 memcpy(second_packet_payload, data.data(), data.size());
868 // The first byte is the header, so we ignore the first byte of |data|.
869 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1,
870 data.size() - 1);
871
872 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
873 .WillOnce(Invoke([kSpatialIndex](EncodedFrame* frame) {
874 EXPECT_EQ(frame->num_references, 0U);
875 EXPECT_EQ(frame->SpatialIndex(), kSpatialIndex);
876 EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u);
877 EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u);
878 EXPECT_THAT(frame->PacketInfos(), SizeIs(2));
879 }));
880
881 rtp_video_stream_receiver_->OnRtpPacket(second_packet);
882 }
883
TEST_F(RtpVideoStreamReceiver2Test,ParseGenericDescriptorRawPayload)884 TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorRawPayload) {
885 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
886 const int kRawPayloadType = 123;
887
888 VideoCodec codec;
889 rtp_video_stream_receiver_->AddReceiveCodec(kRawPayloadType, codec, {},
890 /*raw_payload=*/true);
891 rtp_video_stream_receiver_->StartReceive();
892
893 RtpHeaderExtensionMap extension_map;
894 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
895 RtpPacketReceived rtp_packet(&extension_map);
896
897 RtpGenericFrameDescriptor generic_descriptor;
898 generic_descriptor.SetFirstPacketInSubFrame(true);
899 generic_descriptor.SetLastPacketInSubFrame(true);
900 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
901 generic_descriptor));
902
903 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
904 memcpy(payload, data.data(), data.size());
905 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
906 data.size());
907
908 rtp_packet.SetMarker(true);
909 rtp_packet.SetPayloadType(kRawPayloadType);
910 rtp_packet.SetSequenceNumber(1);
911
912 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
913 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
914 }
915
TEST_F(RtpVideoStreamReceiver2Test,UnwrapsFrameId)916 TEST_F(RtpVideoStreamReceiver2Test, UnwrapsFrameId) {
917 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
918 const int kPayloadType = 123;
919
920 VideoCodec codec;
921 rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, codec, {},
922 /*raw_payload=*/true);
923 rtp_video_stream_receiver_->StartReceive();
924 RtpHeaderExtensionMap extension_map;
925 extension_map.Register<RtpGenericFrameDescriptorExtension00>(5);
926
927 uint16_t rtp_sequence_number = 1;
928 auto inject_packet = [&](uint16_t wrapped_frame_id) {
929 RtpPacketReceived rtp_packet(&extension_map);
930
931 RtpGenericFrameDescriptor generic_descriptor;
932 generic_descriptor.SetFirstPacketInSubFrame(true);
933 generic_descriptor.SetLastPacketInSubFrame(true);
934 generic_descriptor.SetFrameId(wrapped_frame_id);
935 ASSERT_TRUE(rtp_packet.SetExtension<RtpGenericFrameDescriptorExtension00>(
936 generic_descriptor));
937
938 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
939 ASSERT_TRUE(payload);
940 memcpy(payload, data.data(), data.size());
941 mock_on_complete_frame_callback_.ClearExpectedBitstream();
942 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
943 data.size());
944 rtp_packet.SetMarker(true);
945 rtp_packet.SetPayloadType(kPayloadType);
946 rtp_packet.SetSequenceNumber(++rtp_sequence_number);
947 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
948 };
949
950 int64_t first_picture_id;
951 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
952 .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); });
953 inject_packet(/*wrapped_frame_id=*/0xffff);
954
955 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
956 .WillOnce([&](EncodedFrame* frame) {
957 EXPECT_EQ(frame->Id() - first_picture_id, 3);
958 });
959 inject_packet(/*wrapped_frame_id=*/0x0002);
960 }
961
962 class RtpVideoStreamReceiver2DependencyDescriptorTest
963 : public RtpVideoStreamReceiver2Test {
964 public:
RtpVideoStreamReceiver2DependencyDescriptorTest()965 RtpVideoStreamReceiver2DependencyDescriptorTest() {
966 VideoCodec codec;
967 rtp_video_stream_receiver_->AddReceiveCodec(payload_type_, codec, {},
968 /*raw_payload=*/true);
969 extension_map_.Register<RtpDependencyDescriptorExtension>(7);
970 rtp_video_stream_receiver_->StartReceive();
971 }
972
973 // Returns some valid structure for the DependencyDescriptors.
974 // First template of that structure always fit for a key frame.
CreateStreamStructure()975 static FrameDependencyStructure CreateStreamStructure() {
976 FrameDependencyStructure stream_structure;
977 stream_structure.num_decode_targets = 1;
978 stream_structure.templates = {
979 FrameDependencyTemplate().Dtis("S"),
980 FrameDependencyTemplate().Dtis("S").FrameDiffs({1}),
981 };
982 return stream_structure;
983 }
984
InjectPacketWith(const FrameDependencyStructure & stream_structure,const DependencyDescriptor & dependency_descriptor)985 void InjectPacketWith(const FrameDependencyStructure& stream_structure,
986 const DependencyDescriptor& dependency_descriptor) {
987 const std::vector<uint8_t> data = {0, 1, 2, 3, 4};
988 RtpPacketReceived rtp_packet(&extension_map_);
989 ASSERT_TRUE(rtp_packet.SetExtension<RtpDependencyDescriptorExtension>(
990 stream_structure, dependency_descriptor));
991 uint8_t* payload = rtp_packet.SetPayloadSize(data.size());
992 ASSERT_TRUE(payload);
993 memcpy(payload, data.data(), data.size());
994 mock_on_complete_frame_callback_.ClearExpectedBitstream();
995 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
996 data.size());
997 rtp_packet.SetMarker(true);
998 rtp_packet.SetPayloadType(payload_type_);
999 rtp_packet.SetSequenceNumber(++rtp_sequence_number_);
1000 rtp_video_stream_receiver_->OnRtpPacket(rtp_packet);
1001 }
1002
1003 private:
1004 const int payload_type_ = 123;
1005 RtpHeaderExtensionMap extension_map_;
1006 uint16_t rtp_sequence_number_ = 321;
1007 };
1008
TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,UnwrapsFrameId)1009 TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest, UnwrapsFrameId) {
1010 FrameDependencyStructure stream_structure = CreateStreamStructure();
1011
1012 DependencyDescriptor keyframe_descriptor;
1013 keyframe_descriptor.attached_structure =
1014 std::make_unique<FrameDependencyStructure>(stream_structure);
1015 keyframe_descriptor.frame_dependencies = stream_structure.templates[0];
1016 keyframe_descriptor.frame_number = 0xfff0;
1017 // DependencyDescriptor doesn't support reordering delta frame before
1018 // keyframe. Thus feed a key frame first, then test reodered delta frames.
1019 int64_t first_picture_id;
1020 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1021 .WillOnce([&](EncodedFrame* frame) { first_picture_id = frame->Id(); });
1022 InjectPacketWith(stream_structure, keyframe_descriptor);
1023
1024 DependencyDescriptor deltaframe1_descriptor;
1025 deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
1026 deltaframe1_descriptor.frame_number = 0xfffe;
1027
1028 DependencyDescriptor deltaframe2_descriptor;
1029 deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1];
1030 deltaframe2_descriptor.frame_number = 0x0002;
1031
1032 // Parser should unwrap frame ids correctly even if packets were reordered by
1033 // the network.
1034 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1035 .WillOnce([&](EncodedFrame* frame) {
1036 // 0x0002 - 0xfff0
1037 EXPECT_EQ(frame->Id() - first_picture_id, 18);
1038 })
1039 .WillOnce([&](EncodedFrame* frame) {
1040 // 0xfffe - 0xfff0
1041 EXPECT_EQ(frame->Id() - first_picture_id, 14);
1042 });
1043 InjectPacketWith(stream_structure, deltaframe2_descriptor);
1044 InjectPacketWith(stream_structure, deltaframe1_descriptor);
1045 }
1046
TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,DropsLateDeltaFramePacketWithDependencyDescriptorExtension)1047 TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,
1048 DropsLateDeltaFramePacketWithDependencyDescriptorExtension) {
1049 FrameDependencyStructure stream_structure1 = CreateStreamStructure();
1050 FrameDependencyStructure stream_structure2 = CreateStreamStructure();
1051 // Make sure template ids for these two structures do not collide:
1052 // adjust structure_id (that is also used as template id offset).
1053 stream_structure1.structure_id = 13;
1054 stream_structure2.structure_id =
1055 stream_structure1.structure_id + stream_structure1.templates.size();
1056
1057 DependencyDescriptor keyframe1_descriptor;
1058 keyframe1_descriptor.attached_structure =
1059 std::make_unique<FrameDependencyStructure>(stream_structure1);
1060 keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
1061 keyframe1_descriptor.frame_number = 1;
1062 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
1063 InjectPacketWith(stream_structure1, keyframe1_descriptor);
1064
1065 // Pass in 2nd key frame with different structure.
1066 DependencyDescriptor keyframe2_descriptor;
1067 keyframe2_descriptor.attached_structure =
1068 std::make_unique<FrameDependencyStructure>(stream_structure2);
1069 keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
1070 keyframe2_descriptor.frame_number = 3;
1071 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame);
1072 InjectPacketWith(stream_structure2, keyframe2_descriptor);
1073
1074 // Pass in late delta frame that uses structure of the 1st key frame.
1075 DependencyDescriptor deltaframe_descriptor;
1076 deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0];
1077 deltaframe_descriptor.frame_number = 2;
1078 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0);
1079 InjectPacketWith(stream_structure1, deltaframe_descriptor);
1080 }
1081
TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,DropsLateKeyFramePacketWithDependencyDescriptorExtension)1082 TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest,
1083 DropsLateKeyFramePacketWithDependencyDescriptorExtension) {
1084 FrameDependencyStructure stream_structure1 = CreateStreamStructure();
1085 FrameDependencyStructure stream_structure2 = CreateStreamStructure();
1086 // Make sure template ids for these two structures do not collide:
1087 // adjust structure_id (that is also used as template id offset).
1088 stream_structure1.structure_id = 13;
1089 stream_structure2.structure_id =
1090 stream_structure1.structure_id + stream_structure1.templates.size();
1091
1092 DependencyDescriptor keyframe1_descriptor;
1093 keyframe1_descriptor.attached_structure =
1094 std::make_unique<FrameDependencyStructure>(stream_structure1);
1095 keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0];
1096 keyframe1_descriptor.frame_number = 1;
1097
1098 DependencyDescriptor keyframe2_descriptor;
1099 keyframe2_descriptor.attached_structure =
1100 std::make_unique<FrameDependencyStructure>(stream_structure2);
1101 keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0];
1102 keyframe2_descriptor.frame_number = 3;
1103
1104 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1105 .WillOnce(
1106 [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 3); });
1107 InjectPacketWith(stream_structure2, keyframe2_descriptor);
1108 InjectPacketWith(stream_structure1, keyframe1_descriptor);
1109
1110 // Pass in delta frame that uses structure of the 2nd key frame. Late key
1111 // frame shouldn't block it.
1112 DependencyDescriptor deltaframe_descriptor;
1113 deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0];
1114 deltaframe_descriptor.frame_number = 4;
1115 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame)
1116 .WillOnce(
1117 [&](EncodedFrame* frame) { EXPECT_EQ(frame->Id() & 0xFFFF, 4); });
1118 InjectPacketWith(stream_structure2, deltaframe_descriptor);
1119 }
1120
TEST_F(RtpVideoStreamReceiver2Test,TransformFrame)1121 TEST_F(RtpVideoStreamReceiver2Test, TransformFrame) {
1122 rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
1123 new rtc::RefCountedObject<testing::NiceMock<MockFrameTransformer>>();
1124 EXPECT_CALL(*mock_frame_transformer,
1125 RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc));
1126 auto receiver = std::make_unique<RtpVideoStreamReceiver2>(
1127 TaskQueueBase::Current(), Clock::GetRealTimeClock(), &mock_transport_,
1128 nullptr, nullptr, &config_, rtp_receive_statistics_.get(), nullptr,
1129 nullptr, process_thread_.get(), &mock_nack_sender_, nullptr,
1130 &mock_on_complete_frame_callback_, nullptr, mock_frame_transformer);
1131 VideoCodec video_codec;
1132 video_codec.codecType = kVideoCodecGeneric;
1133 receiver->AddReceiveCodec(kPayloadType, video_codec, {},
1134 /*raw_payload=*/false);
1135
1136 RtpPacketReceived rtp_packet;
1137 rtp_packet.SetPayloadType(kPayloadType);
1138 rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
1139 rtp_packet.SetSequenceNumber(1);
1140 RTPVideoHeader video_header =
1141 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
1142 mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
1143 data.size());
1144 EXPECT_CALL(*mock_frame_transformer, Transform(_));
1145 receiver->OnReceivedPayloadData(data, rtp_packet, video_header);
1146
1147 EXPECT_CALL(*mock_frame_transformer,
1148 UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc));
1149 receiver = nullptr;
1150 }
1151
1152 // Test default behavior and when playout delay is overridden by field trial.
1153 const VideoPlayoutDelay kTransmittedPlayoutDelay = {100, 200};
1154 const VideoPlayoutDelay kForcedPlayoutDelay = {70, 90};
1155 struct PlayoutDelayOptions {
1156 std::string field_trial;
1157 VideoPlayoutDelay expected_delay;
1158 };
1159 const PlayoutDelayOptions kDefaultBehavior = {
1160 /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay};
1161 const PlayoutDelayOptions kOverridePlayoutDelay = {
1162 /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/",
1163 /*expected_delay=*/kForcedPlayoutDelay};
1164
1165 class RtpVideoStreamReceiver2TestPlayoutDelay
1166 : public RtpVideoStreamReceiver2Test,
1167 public ::testing::WithParamInterface<PlayoutDelayOptions> {
1168 protected:
RtpVideoStreamReceiver2TestPlayoutDelay()1169 RtpVideoStreamReceiver2TestPlayoutDelay()
1170 : RtpVideoStreamReceiver2Test(GetParam().field_trial) {}
1171 };
1172
1173 INSTANTIATE_TEST_SUITE_P(PlayoutDelay,
1174 RtpVideoStreamReceiver2TestPlayoutDelay,
1175 Values(kDefaultBehavior, kOverridePlayoutDelay));
1176
TEST_P(RtpVideoStreamReceiver2TestPlayoutDelay,PlayoutDelay)1177 TEST_P(RtpVideoStreamReceiver2TestPlayoutDelay, PlayoutDelay) {
1178 rtc::CopyOnWriteBuffer payload_data({1, 2, 3, 4});
1179 RtpHeaderExtensionMap extension_map;
1180 extension_map.Register<PlayoutDelayLimits>(1);
1181 RtpPacketToSend packet_to_send(&extension_map);
1182 packet_to_send.SetPayloadType(kPayloadType);
1183 packet_to_send.SetSequenceNumber(1);
1184
1185 // Set playout delay on outgoing packet.
1186 EXPECT_TRUE(packet_to_send.SetExtension<PlayoutDelayLimits>(
1187 kTransmittedPlayoutDelay));
1188 uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size());
1189 memcpy(payload, payload_data.data(), payload_data.size());
1190
1191 RtpPacketReceived received_packet(&extension_map);
1192 received_packet.Parse(packet_to_send.data(), packet_to_send.size());
1193
1194 RTPVideoHeader video_header =
1195 GetGenericVideoHeader(VideoFrameType::kVideoFrameKey);
1196 mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(),
1197 payload_data.size());
1198 // Expect the playout delay of encoded frame to be the same as the transmitted
1199 // playout delay unless it was overridden by a field trial.
1200 EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_))
1201 .WillOnce(Invoke([expected_playout_delay =
1202 GetParam().expected_delay](EncodedFrame* frame) {
1203 EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay);
1204 }));
1205 rtp_video_stream_receiver_->OnReceivedPayloadData(
1206 received_packet.PayloadBuffer(), received_packet, video_header);
1207 }
1208
1209 } // namespace webrtc
1210