1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/modules/audio_coding/acm2/acm_receiver.h"
12
13 #include <stdlib.h> // malloc
14
15 #include <algorithm> // sort
16 #include <vector>
17
18 #include "webrtc/base/checks.h"
19 #include "webrtc/base/format_macros.h"
20 #include "webrtc/base/logging.h"
21 #include "webrtc/base/safe_conversions.h"
22 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
23 #include "webrtc/common_types.h"
24 #include "webrtc/modules/audio_coding/codecs/audio_decoder.h"
25 #include "webrtc/modules/audio_coding/acm2/acm_resampler.h"
26 #include "webrtc/modules/audio_coding/acm2/call_statistics.h"
27 #include "webrtc/modules/audio_coding/neteq/include/neteq.h"
28 #include "webrtc/system_wrappers/include/clock.h"
29 #include "webrtc/system_wrappers/include/trace.h"
30
31 namespace webrtc {
32
33 namespace acm2 {
34
AcmReceiver(const AudioCodingModule::Config & config)35 AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config)
36 : last_audio_buffer_(new int16_t[AudioFrame::kMaxDataSizeSamples]),
37 neteq_(NetEq::Create(config.neteq_config, config.decoder_factory)),
38 clock_(config.clock),
39 resampled_last_output_frame_(true) {
40 assert(clock_);
41 memset(last_audio_buffer_.get(), 0, AudioFrame::kMaxDataSizeSamples);
42 }
43
~AcmReceiver()44 AcmReceiver::~AcmReceiver() {
45 delete neteq_;
46 }
47
SetMinimumDelay(int delay_ms)48 int AcmReceiver::SetMinimumDelay(int delay_ms) {
49 if (neteq_->SetMinimumDelay(delay_ms))
50 return 0;
51 LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms;
52 return -1;
53 }
54
SetMaximumDelay(int delay_ms)55 int AcmReceiver::SetMaximumDelay(int delay_ms) {
56 if (neteq_->SetMaximumDelay(delay_ms))
57 return 0;
58 LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms;
59 return -1;
60 }
61
LeastRequiredDelayMs() const62 int AcmReceiver::LeastRequiredDelayMs() const {
63 return neteq_->LeastRequiredDelayMs();
64 }
65
last_packet_sample_rate_hz() const66 rtc::Optional<int> AcmReceiver::last_packet_sample_rate_hz() const {
67 rtc::CritScope lock(&crit_sect_);
68 return last_packet_sample_rate_hz_;
69 }
70
last_output_sample_rate_hz() const71 int AcmReceiver::last_output_sample_rate_hz() const {
72 return neteq_->last_output_sample_rate_hz();
73 }
74
InsertPacket(const WebRtcRTPHeader & rtp_header,rtc::ArrayView<const uint8_t> incoming_payload)75 int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header,
76 rtc::ArrayView<const uint8_t> incoming_payload) {
77 uint32_t receive_timestamp = 0;
78 const RTPHeader* header = &rtp_header.header; // Just a shorthand.
79
80 {
81 rtc::CritScope lock(&crit_sect_);
82
83 const rtc::Optional<CodecInst> ci =
84 RtpHeaderToDecoder(*header, incoming_payload[0]);
85 if (!ci) {
86 LOG_F(LS_ERROR) << "Payload-type "
87 << static_cast<int>(header->payloadType)
88 << " is not registered.";
89 return -1;
90 }
91 receive_timestamp = NowInTimestamp(ci->plfreq);
92
93 if (STR_CASE_CMP(ci->plname, "cn") == 0) {
94 if (last_audio_decoder_ && last_audio_decoder_->channels > 1) {
95 // This is a CNG and the audio codec is not mono, so skip pushing in
96 // packets into NetEq.
97 return 0;
98 }
99 } else {
100 last_audio_decoder_ = ci;
101 last_audio_format_ = neteq_->GetDecoderFormat(ci->pltype);
102 RTC_DCHECK(last_audio_format_);
103 last_packet_sample_rate_hz_ = rtc::Optional<int>(ci->plfreq);
104 }
105 } // |crit_sect_| is released.
106
107 if (neteq_->InsertPacket(rtp_header, incoming_payload, receive_timestamp) <
108 0) {
109 LOG(LERROR) << "AcmReceiver::InsertPacket "
110 << static_cast<int>(header->payloadType)
111 << " Failed to insert packet";
112 return -1;
113 }
114 return 0;
115 }
116
GetAudio(int desired_freq_hz,AudioFrame * audio_frame,bool * muted)117 int AcmReceiver::GetAudio(int desired_freq_hz,
118 AudioFrame* audio_frame,
119 bool* muted) {
120 RTC_DCHECK(muted);
121 // Accessing members, take the lock.
122 rtc::CritScope lock(&crit_sect_);
123
124 if (neteq_->GetAudio(audio_frame, muted) != NetEq::kOK) {
125 LOG(LERROR) << "AcmReceiver::GetAudio - NetEq Failed.";
126 return -1;
127 }
128
129 const int current_sample_rate_hz = neteq_->last_output_sample_rate_hz();
130
131 // Update if resampling is required.
132 const bool need_resampling =
133 (desired_freq_hz != -1) && (current_sample_rate_hz != desired_freq_hz);
134
135 if (need_resampling && !resampled_last_output_frame_) {
136 // Prime the resampler with the last frame.
137 int16_t temp_output[AudioFrame::kMaxDataSizeSamples];
138 int samples_per_channel_int = resampler_.Resample10Msec(
139 last_audio_buffer_.get(), current_sample_rate_hz, desired_freq_hz,
140 audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples,
141 temp_output);
142 if (samples_per_channel_int < 0) {
143 LOG(LERROR) << "AcmReceiver::GetAudio - "
144 "Resampling last_audio_buffer_ failed.";
145 return -1;
146 }
147 }
148
149 // TODO(henrik.lundin) Glitches in the output may appear if the output rate
150 // from NetEq changes. See WebRTC issue 3923.
151 if (need_resampling) {
152 int samples_per_channel_int = resampler_.Resample10Msec(
153 audio_frame->data_, current_sample_rate_hz, desired_freq_hz,
154 audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples,
155 audio_frame->data_);
156 if (samples_per_channel_int < 0) {
157 LOG(LERROR) << "AcmReceiver::GetAudio - Resampling audio_buffer_ failed.";
158 return -1;
159 }
160 audio_frame->samples_per_channel_ =
161 static_cast<size_t>(samples_per_channel_int);
162 audio_frame->sample_rate_hz_ = desired_freq_hz;
163 RTC_DCHECK_EQ(
164 audio_frame->sample_rate_hz_,
165 rtc::checked_cast<int>(audio_frame->samples_per_channel_ * 100));
166 resampled_last_output_frame_ = true;
167 } else {
168 resampled_last_output_frame_ = false;
169 // We might end up here ONLY if codec is changed.
170 }
171
172 // Store current audio in |last_audio_buffer_| for next time.
173 memcpy(last_audio_buffer_.get(), audio_frame->data_,
174 sizeof(int16_t) * audio_frame->samples_per_channel_ *
175 audio_frame->num_channels_);
176
177 call_stats_.DecodedByNetEq(audio_frame->speech_type_, *muted);
178 return 0;
179 }
180
AddCodec(int acm_codec_id,uint8_t payload_type,size_t channels,int,AudioDecoder * audio_decoder,const std::string & name)181 int32_t AcmReceiver::AddCodec(int acm_codec_id,
182 uint8_t payload_type,
183 size_t channels,
184 int /*sample_rate_hz*/,
185 AudioDecoder* audio_decoder,
186 const std::string& name) {
187 // TODO(kwiberg): This function has been ignoring the |sample_rate_hz|
188 // argument for a long time. Arguably, it should simply be removed.
189
190 const auto neteq_decoder = [acm_codec_id, channels]() -> NetEqDecoder {
191 if (acm_codec_id == -1)
192 return NetEqDecoder::kDecoderArbitrary; // External decoder.
193 const rtc::Optional<RentACodec::CodecId> cid =
194 RentACodec::CodecIdFromIndex(acm_codec_id);
195 RTC_DCHECK(cid) << "Invalid codec index: " << acm_codec_id;
196 const rtc::Optional<NetEqDecoder> ned =
197 RentACodec::NetEqDecoderFromCodecId(*cid, channels);
198 RTC_DCHECK(ned) << "Invalid codec ID: " << static_cast<int>(*cid);
199 return *ned;
200 }();
201 const rtc::Optional<SdpAudioFormat> new_format =
202 RentACodec::NetEqDecoderToSdpAudioFormat(neteq_decoder);
203
204 rtc::CritScope lock(&crit_sect_);
205
206 const auto old_format = neteq_->GetDecoderFormat(payload_type);
207 if (old_format && new_format && *old_format == *new_format) {
208 // Re-registering the same codec. Do nothing and return.
209 return 0;
210 }
211
212 if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK &&
213 neteq_->LastError() != NetEq::kDecoderNotFound) {
214 LOG(LERROR) << "Cannot remove payload " << static_cast<int>(payload_type);
215 return -1;
216 }
217
218 int ret_val;
219 if (!audio_decoder) {
220 ret_val = neteq_->RegisterPayloadType(neteq_decoder, name, payload_type);
221 } else {
222 ret_val = neteq_->RegisterExternalDecoder(
223 audio_decoder, neteq_decoder, name, payload_type);
224 }
225 if (ret_val != NetEq::kOK) {
226 LOG(LERROR) << "AcmReceiver::AddCodec " << acm_codec_id
227 << static_cast<int>(payload_type)
228 << " channels: " << channels;
229 return -1;
230 }
231 return 0;
232 }
233
AddCodec(int rtp_payload_type,const SdpAudioFormat & audio_format)234 bool AcmReceiver::AddCodec(int rtp_payload_type,
235 const SdpAudioFormat& audio_format) {
236 const auto old_format = neteq_->GetDecoderFormat(rtp_payload_type);
237 if (old_format && *old_format == audio_format) {
238 // Re-registering the same codec. Do nothing and return.
239 return true;
240 }
241
242 if (neteq_->RemovePayloadType(rtp_payload_type) != NetEq::kOK &&
243 neteq_->LastError() != NetEq::kDecoderNotFound) {
244 LOG(LERROR) << "AcmReceiver::AddCodec: Could not remove existing decoder"
245 " for payload type "
246 << rtp_payload_type;
247 return false;
248 }
249
250 const bool success =
251 neteq_->RegisterPayloadType(rtp_payload_type, audio_format);
252 if (!success) {
253 LOG(LERROR) << "AcmReceiver::AddCodec failed for payload type "
254 << rtp_payload_type << ", decoder format " << audio_format;
255 }
256 return success;
257 }
258
FlushBuffers()259 void AcmReceiver::FlushBuffers() {
260 neteq_->FlushBuffers();
261 }
262
RemoveAllCodecs()263 void AcmReceiver::RemoveAllCodecs() {
264 rtc::CritScope lock(&crit_sect_);
265 neteq_->RemoveAllPayloadTypes();
266 last_audio_decoder_ = rtc::Optional<CodecInst>();
267 last_audio_format_ = rtc::Optional<SdpAudioFormat>();
268 last_packet_sample_rate_hz_ = rtc::Optional<int>();
269 }
270
RemoveCodec(uint8_t payload_type)271 int AcmReceiver::RemoveCodec(uint8_t payload_type) {
272 rtc::CritScope lock(&crit_sect_);
273 if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK &&
274 neteq_->LastError() != NetEq::kDecoderNotFound) {
275 LOG(LERROR) << "AcmReceiver::RemoveCodec" << static_cast<int>(payload_type);
276 return -1;
277 }
278 if (last_audio_decoder_ && payload_type == last_audio_decoder_->pltype) {
279 last_audio_decoder_ = rtc::Optional<CodecInst>();
280 last_audio_format_ = rtc::Optional<SdpAudioFormat>();
281 last_packet_sample_rate_hz_ = rtc::Optional<int>();
282 }
283 return 0;
284 }
285
GetPlayoutTimestamp()286 rtc::Optional<uint32_t> AcmReceiver::GetPlayoutTimestamp() {
287 return neteq_->GetPlayoutTimestamp();
288 }
289
FilteredCurrentDelayMs() const290 int AcmReceiver::FilteredCurrentDelayMs() const {
291 return neteq_->FilteredCurrentDelayMs();
292 }
293
LastAudioCodec(CodecInst * codec) const294 int AcmReceiver::LastAudioCodec(CodecInst* codec) const {
295 rtc::CritScope lock(&crit_sect_);
296 if (!last_audio_decoder_) {
297 return -1;
298 }
299 *codec = *last_audio_decoder_;
300 return 0;
301 }
302
LastAudioFormat() const303 rtc::Optional<SdpAudioFormat> AcmReceiver::LastAudioFormat() const {
304 rtc::CritScope lock(&crit_sect_);
305 return last_audio_format_;
306 }
307
GetNetworkStatistics(NetworkStatistics * acm_stat)308 void AcmReceiver::GetNetworkStatistics(NetworkStatistics* acm_stat) {
309 NetEqNetworkStatistics neteq_stat;
310 // NetEq function always returns zero, so we don't check the return value.
311 neteq_->NetworkStatistics(&neteq_stat);
312
313 acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms;
314 acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms;
315 acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false;
316 acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate;
317 acm_stat->currentDiscardRate = neteq_stat.packet_discard_rate;
318 acm_stat->currentExpandRate = neteq_stat.expand_rate;
319 acm_stat->currentSpeechExpandRate = neteq_stat.speech_expand_rate;
320 acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate;
321 acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate;
322 acm_stat->currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate;
323 acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm;
324 acm_stat->addedSamples = neteq_stat.added_zero_samples;
325 acm_stat->meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms;
326 acm_stat->medianWaitingTimeMs = neteq_stat.median_waiting_time_ms;
327 acm_stat->minWaitingTimeMs = neteq_stat.min_waiting_time_ms;
328 acm_stat->maxWaitingTimeMs = neteq_stat.max_waiting_time_ms;
329 }
330
DecoderByPayloadType(uint8_t payload_type,CodecInst * codec) const331 int AcmReceiver::DecoderByPayloadType(uint8_t payload_type,
332 CodecInst* codec) const {
333 rtc::CritScope lock(&crit_sect_);
334 const rtc::Optional<CodecInst> ci = neteq_->GetDecoder(payload_type);
335 if (ci) {
336 *codec = *ci;
337 return 0;
338 } else {
339 LOG(LERROR) << "AcmReceiver::DecoderByPayloadType "
340 << static_cast<int>(payload_type);
341 return -1;
342 }
343 }
344
EnableNack(size_t max_nack_list_size)345 int AcmReceiver::EnableNack(size_t max_nack_list_size) {
346 neteq_->EnableNack(max_nack_list_size);
347 return 0;
348 }
349
DisableNack()350 void AcmReceiver::DisableNack() {
351 neteq_->DisableNack();
352 }
353
GetNackList(int64_t round_trip_time_ms) const354 std::vector<uint16_t> AcmReceiver::GetNackList(
355 int64_t round_trip_time_ms) const {
356 return neteq_->GetNackList(round_trip_time_ms);
357 }
358
ResetInitialDelay()359 void AcmReceiver::ResetInitialDelay() {
360 neteq_->SetMinimumDelay(0);
361 // TODO(turajs): Should NetEq Buffer be flushed?
362 }
363
RtpHeaderToDecoder(const RTPHeader & rtp_header,uint8_t first_payload_byte) const364 const rtc::Optional<CodecInst> AcmReceiver::RtpHeaderToDecoder(
365 const RTPHeader& rtp_header,
366 uint8_t first_payload_byte) const {
367 const rtc::Optional<CodecInst> ci =
368 neteq_->GetDecoder(rtp_header.payloadType);
369 if (ci && STR_CASE_CMP(ci->plname, "red") == 0) {
370 // This is a RED packet. Get the payload of the audio codec.
371 return neteq_->GetDecoder(first_payload_byte & 0x7f);
372 } else {
373 return ci;
374 }
375 }
376
NowInTimestamp(int decoder_sampling_rate) const377 uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const {
378 // Down-cast the time to (32-6)-bit since we only care about
379 // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
380 // We masked 6 most significant bits of 32-bit so there is no overflow in
381 // the conversion from milliseconds to timestamp.
382 const uint32_t now_in_ms = static_cast<uint32_t>(
383 clock_->TimeInMilliseconds() & 0x03ffffff);
384 return static_cast<uint32_t>(
385 (decoder_sampling_rate / 1000) * now_in_ms);
386 }
387
GetDecodingCallStatistics(AudioDecodingCallStats * stats) const388 void AcmReceiver::GetDecodingCallStatistics(
389 AudioDecodingCallStats* stats) const {
390 rtc::CritScope lock(&crit_sect_);
391 *stats = call_stats_.GetDecodingStatistics();
392 }
393
394 } // namespace acm2
395
396 } // namespace webrtc
397