1 /*
2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/neteq/test/neteq_decoding_test.h"
12
13 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
14 #include "api/rtp_headers.h"
15 #include "modules/audio_coding/neteq/default_neteq_factory.h"
16 #include "modules/audio_coding/neteq/test/result_sink.h"
17 #include "rtc_base/strings/string_builder.h"
18 #include "test/testsupport/file_utils.h"
19
20 #ifdef WEBRTC_NETEQ_UNITTEST_BITEXACT
21 RTC_PUSH_IGNORING_WUNDEF()
22 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
23 #include "external/webrtc/webrtc/modules/audio_coding/neteq/neteq_unittest.pb.h"
24 #else
25 #include "modules/audio_coding/neteq/neteq_unittest.pb.h"
26 #endif
27 RTC_POP_IGNORING_WUNDEF()
28 #endif
29
30 namespace webrtc {
31
32 namespace {
33
LoadDecoders(webrtc::NetEq * neteq)34 void LoadDecoders(webrtc::NetEq* neteq) {
35 ASSERT_EQ(true,
36 neteq->RegisterPayloadType(0, SdpAudioFormat("pcmu", 8000, 1)));
37 ASSERT_EQ(true,
38 neteq->RegisterPayloadType(8, SdpAudioFormat("pcma", 8000, 1)));
39 #ifdef WEBRTC_CODEC_ILBC
40 ASSERT_EQ(true,
41 neteq->RegisterPayloadType(102, SdpAudioFormat("ilbc", 8000, 1)));
42 #endif
43 #if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
44 ASSERT_EQ(true,
45 neteq->RegisterPayloadType(103, SdpAudioFormat("isac", 16000, 1)));
46 #endif
47 #ifdef WEBRTC_CODEC_ISAC
48 ASSERT_EQ(true,
49 neteq->RegisterPayloadType(104, SdpAudioFormat("isac", 32000, 1)));
50 #endif
51 #ifdef WEBRTC_CODEC_OPUS
52 ASSERT_EQ(true,
53 neteq->RegisterPayloadType(
54 111, SdpAudioFormat("opus", 48000, 2, {{"stereo", "0"}})));
55 #endif
56 ASSERT_EQ(true,
57 neteq->RegisterPayloadType(93, SdpAudioFormat("L16", 8000, 1)));
58 ASSERT_EQ(true,
59 neteq->RegisterPayloadType(94, SdpAudioFormat("L16", 16000, 1)));
60 ASSERT_EQ(true,
61 neteq->RegisterPayloadType(95, SdpAudioFormat("L16", 32000, 1)));
62 ASSERT_EQ(true,
63 neteq->RegisterPayloadType(13, SdpAudioFormat("cn", 8000, 1)));
64 ASSERT_EQ(true,
65 neteq->RegisterPayloadType(98, SdpAudioFormat("cn", 16000, 1)));
66 }
67
68 } // namespace
69
70 const int NetEqDecodingTest::kTimeStepMs;
71 const size_t NetEqDecodingTest::kBlockSize8kHz;
72 const size_t NetEqDecodingTest::kBlockSize16kHz;
73 const size_t NetEqDecodingTest::kBlockSize32kHz;
74 const int NetEqDecodingTest::kInitSampleRateHz;
75
NetEqDecodingTest()76 NetEqDecodingTest::NetEqDecodingTest()
77 : clock_(0),
78 config_(),
79 output_sample_rate_(kInitSampleRateHz),
80 algorithmic_delay_ms_(0) {
81 config_.sample_rate_hz = kInitSampleRateHz;
82 }
83
SetUp()84 void NetEqDecodingTest::SetUp() {
85 auto decoder_factory = CreateBuiltinAudioDecoderFactory();
86 neteq_ = DefaultNetEqFactory().CreateNetEq(config_, decoder_factory, &clock_);
87 NetEqNetworkStatistics stat;
88 ASSERT_EQ(0, neteq_->NetworkStatistics(&stat));
89 algorithmic_delay_ms_ = stat.current_buffer_size_ms;
90 ASSERT_TRUE(neteq_);
91 LoadDecoders(neteq_.get());
92 }
93
TearDown()94 void NetEqDecodingTest::TearDown() {}
95
OpenInputFile(const std::string & rtp_file)96 void NetEqDecodingTest::OpenInputFile(const std::string& rtp_file) {
97 rtp_source_.reset(test::RtpFileSource::Create(rtp_file));
98 }
99
Process()100 void NetEqDecodingTest::Process() {
101 // Check if time to receive.
102 while (packet_ && clock_.TimeInMilliseconds() >= packet_->time_ms()) {
103 if (packet_->payload_length_bytes() > 0) {
104 #ifndef WEBRTC_CODEC_ISAC
105 // Ignore payload type 104 (iSAC-swb) if ISAC is not supported.
106 if (packet_->header().payloadType != 104)
107 #endif
108 ASSERT_EQ(
109 0, neteq_->InsertPacket(
110 packet_->header(),
111 rtc::ArrayView<const uint8_t>(
112 packet_->payload(), packet_->payload_length_bytes())));
113 }
114 // Get next packet.
115 packet_ = rtp_source_->NextPacket();
116 }
117
118 // Get audio from NetEq.
119 bool muted;
120 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
121 ASSERT_FALSE(muted);
122 ASSERT_TRUE((out_frame_.samples_per_channel_ == kBlockSize8kHz) ||
123 (out_frame_.samples_per_channel_ == kBlockSize16kHz) ||
124 (out_frame_.samples_per_channel_ == kBlockSize32kHz) ||
125 (out_frame_.samples_per_channel_ == kBlockSize48kHz));
126 output_sample_rate_ = out_frame_.sample_rate_hz_;
127 EXPECT_EQ(output_sample_rate_, neteq_->last_output_sample_rate_hz());
128
129 // Increase time.
130 clock_.AdvanceTimeMilliseconds(kTimeStepMs);
131 }
132
DecodeAndCompare(const std::string & rtp_file,const std::string & output_checksum,const std::string & network_stats_checksum,bool gen_ref)133 void NetEqDecodingTest::DecodeAndCompare(
134 const std::string& rtp_file,
135 const std::string& output_checksum,
136 const std::string& network_stats_checksum,
137 bool gen_ref) {
138 OpenInputFile(rtp_file);
139
140 std::string ref_out_file =
141 gen_ref ? webrtc::test::OutputPath() + "neteq_universal_ref.pcm" : "";
142 ResultSink output(ref_out_file);
143
144 std::string stat_out_file =
145 gen_ref ? webrtc::test::OutputPath() + "neteq_network_stats.dat" : "";
146 ResultSink network_stats(stat_out_file);
147
148 packet_ = rtp_source_->NextPacket();
149 int i = 0;
150 uint64_t last_concealed_samples = 0;
151 uint64_t last_total_samples_received = 0;
152 while (packet_) {
153 rtc::StringBuilder ss;
154 ss << "Lap number " << i++ << " in DecodeAndCompare while loop";
155 SCOPED_TRACE(ss.str()); // Print out the parameter values on failure.
156 ASSERT_NO_FATAL_FAILURE(Process());
157 ASSERT_NO_FATAL_FAILURE(
158 output.AddResult(out_frame_.data(), out_frame_.samples_per_channel_));
159
160 // Query the network statistics API once per second
161 if (clock_.TimeInMilliseconds() % 1000 == 0) {
162 // Process NetworkStatistics.
163 NetEqNetworkStatistics current_network_stats;
164 ASSERT_EQ(0, neteq_->NetworkStatistics(¤t_network_stats));
165 ASSERT_NO_FATAL_FAILURE(network_stats.AddResult(current_network_stats));
166
167 // Verify that liftime stats and network stats report similar loss
168 // concealment rates.
169 auto lifetime_stats = neteq_->GetLifetimeStatistics();
170 const uint64_t delta_concealed_samples =
171 lifetime_stats.concealed_samples - last_concealed_samples;
172 last_concealed_samples = lifetime_stats.concealed_samples;
173 const uint64_t delta_total_samples_received =
174 lifetime_stats.total_samples_received - last_total_samples_received;
175 last_total_samples_received = lifetime_stats.total_samples_received;
176 // The tolerance is 1% but expressed in Q14.
177 EXPECT_NEAR(
178 (delta_concealed_samples << 14) / delta_total_samples_received,
179 current_network_stats.expand_rate, (2 << 14) / 100.0);
180 }
181 }
182
183 SCOPED_TRACE("Check output audio.");
184 output.VerifyChecksum(output_checksum);
185 SCOPED_TRACE("Check network stats.");
186 network_stats.VerifyChecksum(network_stats_checksum);
187 }
188
PopulateRtpInfo(int frame_index,int timestamp,RTPHeader * rtp_info)189 void NetEqDecodingTest::PopulateRtpInfo(int frame_index,
190 int timestamp,
191 RTPHeader* rtp_info) {
192 rtp_info->sequenceNumber = frame_index;
193 rtp_info->timestamp = timestamp;
194 rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC.
195 rtp_info->payloadType = 94; // PCM16b WB codec.
196 rtp_info->markerBit = 0;
197 }
198
PopulateCng(int frame_index,int timestamp,RTPHeader * rtp_info,uint8_t * payload,size_t * payload_len)199 void NetEqDecodingTest::PopulateCng(int frame_index,
200 int timestamp,
201 RTPHeader* rtp_info,
202 uint8_t* payload,
203 size_t* payload_len) {
204 rtp_info->sequenceNumber = frame_index;
205 rtp_info->timestamp = timestamp;
206 rtp_info->ssrc = 0x1234; // Just an arbitrary SSRC.
207 rtp_info->payloadType = 98; // WB CNG.
208 rtp_info->markerBit = 0;
209 payload[0] = 64; // Noise level -64 dBov, quite arbitrarily chosen.
210 *payload_len = 1; // Only noise level, no spectral parameters.
211 }
212
WrapTest(uint16_t start_seq_no,uint32_t start_timestamp,const std::set<uint16_t> & drop_seq_numbers,bool expect_seq_no_wrap,bool expect_timestamp_wrap)213 void NetEqDecodingTest::WrapTest(uint16_t start_seq_no,
214 uint32_t start_timestamp,
215 const std::set<uint16_t>& drop_seq_numbers,
216 bool expect_seq_no_wrap,
217 bool expect_timestamp_wrap) {
218 uint16_t seq_no = start_seq_no;
219 uint32_t timestamp = start_timestamp;
220 const int kBlocksPerFrame = 3; // Number of 10 ms blocks per frame.
221 const int kFrameSizeMs = kBlocksPerFrame * kTimeStepMs;
222 const int kSamples = kBlockSize16kHz * kBlocksPerFrame;
223 const size_t kPayloadBytes = kSamples * sizeof(int16_t);
224 double next_input_time_ms = 0.0;
225 uint32_t receive_timestamp = 0;
226
227 // Insert speech for 2 seconds.
228 const int kSpeechDurationMs = 2000;
229 int packets_inserted = 0;
230 uint16_t last_seq_no;
231 uint32_t last_timestamp;
232 bool timestamp_wrapped = false;
233 bool seq_no_wrapped = false;
234 for (double t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) {
235 // Each turn in this for loop is 10 ms.
236 while (next_input_time_ms <= t_ms) {
237 // Insert one 30 ms speech frame.
238 uint8_t payload[kPayloadBytes] = {0};
239 RTPHeader rtp_info;
240 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
241 if (drop_seq_numbers.find(seq_no) == drop_seq_numbers.end()) {
242 // This sequence number was not in the set to drop. Insert it.
243 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
244 ++packets_inserted;
245 }
246 NetEqNetworkStatistics network_stats;
247 ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats));
248
249 // Due to internal NetEq logic, preferred buffer-size is about 4 times the
250 // packet size for first few packets. Therefore we refrain from checking
251 // the criteria.
252 if (packets_inserted > 4) {
253 // Expect preferred and actual buffer size to be no more than 2 frames.
254 EXPECT_LE(network_stats.preferred_buffer_size_ms, kFrameSizeMs * 2);
255 EXPECT_LE(network_stats.current_buffer_size_ms,
256 kFrameSizeMs * 2 + algorithmic_delay_ms_);
257 }
258 last_seq_no = seq_no;
259 last_timestamp = timestamp;
260
261 ++seq_no;
262 timestamp += kSamples;
263 receive_timestamp += kSamples;
264 next_input_time_ms += static_cast<double>(kFrameSizeMs);
265
266 seq_no_wrapped |= seq_no < last_seq_no;
267 timestamp_wrapped |= timestamp < last_timestamp;
268 }
269 // Pull out data once.
270 AudioFrame output;
271 bool muted;
272 ASSERT_EQ(0, neteq_->GetAudio(&output, &muted));
273 ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
274 ASSERT_EQ(1u, output.num_channels_);
275
276 // Expect delay (in samples) to be less than 2 packets.
277 absl::optional<uint32_t> playout_timestamp = neteq_->GetPlayoutTimestamp();
278 ASSERT_TRUE(playout_timestamp);
279 EXPECT_LE(timestamp - *playout_timestamp,
280 static_cast<uint32_t>(kSamples * 2));
281 }
282 // Make sure we have actually tested wrap-around.
283 ASSERT_EQ(expect_seq_no_wrap, seq_no_wrapped);
284 ASSERT_EQ(expect_timestamp_wrap, timestamp_wrapped);
285 }
286
LongCngWithClockDrift(double drift_factor,double network_freeze_ms,bool pull_audio_during_freeze,int delay_tolerance_ms,int max_time_to_speech_ms)287 void NetEqDecodingTest::LongCngWithClockDrift(double drift_factor,
288 double network_freeze_ms,
289 bool pull_audio_during_freeze,
290 int delay_tolerance_ms,
291 int max_time_to_speech_ms) {
292 uint16_t seq_no = 0;
293 uint32_t timestamp = 0;
294 const int kFrameSizeMs = 30;
295 const size_t kSamples = kFrameSizeMs * 16;
296 const size_t kPayloadBytes = kSamples * 2;
297 double next_input_time_ms = 0.0;
298 double t_ms;
299 bool muted;
300
301 // Insert speech for 5 seconds.
302 const int kSpeechDurationMs = 5000;
303 for (t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) {
304 // Each turn in this for loop is 10 ms.
305 while (next_input_time_ms <= t_ms) {
306 // Insert one 30 ms speech frame.
307 uint8_t payload[kPayloadBytes] = {0};
308 RTPHeader rtp_info;
309 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
310 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
311 ++seq_no;
312 timestamp += kSamples;
313 next_input_time_ms += static_cast<double>(kFrameSizeMs) * drift_factor;
314 }
315 // Pull out data once.
316 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
317 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
318 }
319
320 EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
321 absl::optional<uint32_t> playout_timestamp = neteq_->GetPlayoutTimestamp();
322 ASSERT_TRUE(playout_timestamp);
323 int32_t delay_before = timestamp - *playout_timestamp;
324
325 // Insert CNG for 1 minute (= 60000 ms).
326 const int kCngPeriodMs = 100;
327 const int kCngPeriodSamples = kCngPeriodMs * 16; // Period in 16 kHz samples.
328 const int kCngDurationMs = 60000;
329 for (; t_ms < kSpeechDurationMs + kCngDurationMs; t_ms += 10) {
330 // Each turn in this for loop is 10 ms.
331 while (next_input_time_ms <= t_ms) {
332 // Insert one CNG frame each 100 ms.
333 uint8_t payload[kPayloadBytes];
334 size_t payload_len;
335 RTPHeader rtp_info;
336 PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
337 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView<const uint8_t>(
338 payload, payload_len)));
339 ++seq_no;
340 timestamp += kCngPeriodSamples;
341 next_input_time_ms += static_cast<double>(kCngPeriodMs) * drift_factor;
342 }
343 // Pull out data once.
344 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
345 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
346 }
347
348 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
349
350 if (network_freeze_ms > 0) {
351 // First keep pulling audio for |network_freeze_ms| without inserting
352 // any data, then insert CNG data corresponding to |network_freeze_ms|
353 // without pulling any output audio.
354 const double loop_end_time = t_ms + network_freeze_ms;
355 for (; t_ms < loop_end_time; t_ms += 10) {
356 // Pull out data once.
357 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
358 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
359 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
360 }
361 bool pull_once = pull_audio_during_freeze;
362 // If |pull_once| is true, GetAudio will be called once half-way through
363 // the network recovery period.
364 double pull_time_ms = (t_ms + next_input_time_ms) / 2;
365 while (next_input_time_ms <= t_ms) {
366 if (pull_once && next_input_time_ms >= pull_time_ms) {
367 pull_once = false;
368 // Pull out data once.
369 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
370 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
371 EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
372 t_ms += 10;
373 }
374 // Insert one CNG frame each 100 ms.
375 uint8_t payload[kPayloadBytes];
376 size_t payload_len;
377 RTPHeader rtp_info;
378 PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
379 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView<const uint8_t>(
380 payload, payload_len)));
381 ++seq_no;
382 timestamp += kCngPeriodSamples;
383 next_input_time_ms += kCngPeriodMs * drift_factor;
384 }
385 }
386
387 // Insert speech again until output type is speech.
388 double speech_restart_time_ms = t_ms;
389 while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) {
390 // Each turn in this for loop is 10 ms.
391 while (next_input_time_ms <= t_ms) {
392 // Insert one 30 ms speech frame.
393 uint8_t payload[kPayloadBytes] = {0};
394 RTPHeader rtp_info;
395 PopulateRtpInfo(seq_no, timestamp, &rtp_info);
396 ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload));
397 ++seq_no;
398 timestamp += kSamples;
399 next_input_time_ms += kFrameSizeMs * drift_factor;
400 }
401 // Pull out data once.
402 ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted));
403 ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
404 // Increase clock.
405 t_ms += 10;
406 }
407
408 // Check that the speech starts again within reasonable time.
409 double time_until_speech_returns_ms = t_ms - speech_restart_time_ms;
410 EXPECT_LT(time_until_speech_returns_ms, max_time_to_speech_ms);
411 playout_timestamp = neteq_->GetPlayoutTimestamp();
412 ASSERT_TRUE(playout_timestamp);
413 int32_t delay_after = timestamp - *playout_timestamp;
414 // Compare delay before and after, and make sure it differs less than 20 ms.
415 EXPECT_LE(delay_after, delay_before + delay_tolerance_ms * 16);
416 EXPECT_GE(delay_after, delay_before - delay_tolerance_ms * 16);
417 }
418
SetUp()419 void NetEqDecodingTestTwoInstances::SetUp() {
420 NetEqDecodingTest::SetUp();
421 config2_ = config_;
422 }
423
CreateSecondInstance()424 void NetEqDecodingTestTwoInstances::CreateSecondInstance() {
425 auto decoder_factory = CreateBuiltinAudioDecoderFactory();
426 neteq2_ =
427 DefaultNetEqFactory().CreateNetEq(config2_, decoder_factory, &clock_);
428 ASSERT_TRUE(neteq2_);
429 LoadDecoders(neteq2_.get());
430 }
431
432 } // namespace webrtc
433