1 /*
2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "webrtc/modules/audio_coding/neteq/neteq_impl.h"
12 
13 #include <assert.h>
14 #include <memory.h>  // memset
15 
16 #include <algorithm>
17 
18 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
19 #include "webrtc/modules/audio_coding/codecs/audio_decoder.h"
20 #include "webrtc/modules/audio_coding/neteq/accelerate.h"
21 #include "webrtc/modules/audio_coding/neteq/background_noise.h"
22 #include "webrtc/modules/audio_coding/neteq/buffer_level_filter.h"
23 #include "webrtc/modules/audio_coding/neteq/comfort_noise.h"
24 #include "webrtc/modules/audio_coding/neteq/decision_logic.h"
25 #include "webrtc/modules/audio_coding/neteq/decoder_database.h"
26 #include "webrtc/modules/audio_coding/neteq/defines.h"
27 #include "webrtc/modules/audio_coding/neteq/delay_manager.h"
28 #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h"
29 #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h"
30 #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h"
31 #include "webrtc/modules/audio_coding/neteq/expand.h"
32 #include "webrtc/modules/audio_coding/neteq/merge.h"
33 #include "webrtc/modules/audio_coding/neteq/normal.h"
34 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
35 #include "webrtc/modules/audio_coding/neteq/packet.h"
36 #include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
37 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
38 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
39 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
40 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
41 #include "webrtc/modules/interface/module_common_types.h"
42 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
43 #include "webrtc/system_wrappers/interface/logging.h"
44 
45 // Modify the code to obtain backwards bit-exactness. Once bit-exactness is no
46 // longer required, this #define should be removed (and the code that it
47 // enables).
48 #define LEGACY_BITEXACT
49 
50 namespace webrtc {
51 
NetEqImpl(const NetEq::Config & config,BufferLevelFilter * buffer_level_filter,DecoderDatabase * decoder_database,DelayManager * delay_manager,DelayPeakDetector * delay_peak_detector,DtmfBuffer * dtmf_buffer,DtmfToneGenerator * dtmf_tone_generator,PacketBuffer * packet_buffer,PayloadSplitter * payload_splitter,TimestampScaler * timestamp_scaler,AccelerateFactory * accelerate_factory,ExpandFactory * expand_factory,PreemptiveExpandFactory * preemptive_expand_factory,bool create_components)52 NetEqImpl::NetEqImpl(const NetEq::Config& config,
53                      BufferLevelFilter* buffer_level_filter,
54                      DecoderDatabase* decoder_database,
55                      DelayManager* delay_manager,
56                      DelayPeakDetector* delay_peak_detector,
57                      DtmfBuffer* dtmf_buffer,
58                      DtmfToneGenerator* dtmf_tone_generator,
59                      PacketBuffer* packet_buffer,
60                      PayloadSplitter* payload_splitter,
61                      TimestampScaler* timestamp_scaler,
62                      AccelerateFactory* accelerate_factory,
63                      ExpandFactory* expand_factory,
64                      PreemptiveExpandFactory* preemptive_expand_factory,
65                      bool create_components)
66     : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
67       buffer_level_filter_(buffer_level_filter),
68       decoder_database_(decoder_database),
69       delay_manager_(delay_manager),
70       delay_peak_detector_(delay_peak_detector),
71       dtmf_buffer_(dtmf_buffer),
72       dtmf_tone_generator_(dtmf_tone_generator),
73       packet_buffer_(packet_buffer),
74       payload_splitter_(payload_splitter),
75       timestamp_scaler_(timestamp_scaler),
76       vad_(new PostDecodeVad()),
77       expand_factory_(expand_factory),
78       accelerate_factory_(accelerate_factory),
79       preemptive_expand_factory_(preemptive_expand_factory),
80       last_mode_(kModeNormal),
81       decoded_buffer_length_(kMaxFrameSize),
82       decoded_buffer_(new int16_t[decoded_buffer_length_]),
83       playout_timestamp_(0),
84       new_codec_(false),
85       timestamp_(0),
86       reset_decoder_(false),
87       current_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
88       current_cng_rtp_payload_type_(0xFF),  // Invalid RTP payload type.
89       ssrc_(0),
90       first_packet_(true),
91       error_code_(0),
92       decoder_error_code_(0),
93       background_noise_mode_(config.background_noise_mode),
94       playout_mode_(config.playout_mode),
95       decoded_packet_sequence_number_(-1),
96       decoded_packet_timestamp_(0) {
97   int fs = config.sample_rate_hz;
98   if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) {
99     LOG(LS_ERROR) << "Sample rate " << fs << " Hz not supported. " <<
100         "Changing to 8000 Hz.";
101     fs = 8000;
102   }
103   LOG(LS_VERBOSE) << "Create NetEqImpl object with fs = " << fs << ".";
104   fs_hz_ = fs;
105   fs_mult_ = fs / 8000;
106   output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
107   decoder_frame_length_ = 3 * output_size_samples_;
108   WebRtcSpl_Init();
109   if (create_components) {
110     SetSampleRateAndChannels(fs, 1);  // Default is 1 channel.
111   }
112 }
113 
~NetEqImpl()114 NetEqImpl::~NetEqImpl() {
115   LOG(LS_INFO) << "Deleting NetEqImpl object.";
116 }
117 
InsertPacket(const WebRtcRTPHeader & rtp_header,const uint8_t * payload,size_t length_bytes,uint32_t receive_timestamp)118 int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header,
119                             const uint8_t* payload,
120                             size_t length_bytes,
121                             uint32_t receive_timestamp) {
122   CriticalSectionScoped lock(crit_sect_.get());
123   LOG(LS_VERBOSE) << "InsertPacket: ts=" << rtp_header.header.timestamp <<
124       ", sn=" << rtp_header.header.sequenceNumber <<
125       ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
126       ", ssrc=" << rtp_header.header.ssrc <<
127       ", len=" << length_bytes;
128   int error = InsertPacketInternal(rtp_header, payload, length_bytes,
129                                    receive_timestamp, false);
130   if (error != 0) {
131     LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
132     error_code_ = error;
133     return kFail;
134   }
135   return kOK;
136 }
137 
InsertSyncPacket(const WebRtcRTPHeader & rtp_header,uint32_t receive_timestamp)138 int NetEqImpl::InsertSyncPacket(const WebRtcRTPHeader& rtp_header,
139                                 uint32_t receive_timestamp) {
140   CriticalSectionScoped lock(crit_sect_.get());
141   LOG(LS_VERBOSE) << "InsertPacket-Sync: ts="
142       << rtp_header.header.timestamp <<
143       ", sn=" << rtp_header.header.sequenceNumber <<
144       ", pt=" << static_cast<int>(rtp_header.header.payloadType) <<
145       ", ssrc=" << rtp_header.header.ssrc;
146 
147   const uint8_t kSyncPayload[] = { 's', 'y', 'n', 'c' };
148   int error = InsertPacketInternal(
149       rtp_header, kSyncPayload, sizeof(kSyncPayload), receive_timestamp, true);
150 
151   if (error != 0) {
152     LOG_FERR1(LS_WARNING, InsertPacketInternal, error);
153     error_code_ = error;
154     return kFail;
155   }
156   return kOK;
157 }
158 
GetAudio(size_t max_length,int16_t * output_audio,int * samples_per_channel,int * num_channels,NetEqOutputType * type)159 int NetEqImpl::GetAudio(size_t max_length, int16_t* output_audio,
160                         int* samples_per_channel, int* num_channels,
161                         NetEqOutputType* type) {
162   CriticalSectionScoped lock(crit_sect_.get());
163   LOG(LS_VERBOSE) << "GetAudio";
164   int error = GetAudioInternal(max_length, output_audio, samples_per_channel,
165                                num_channels);
166   LOG(LS_VERBOSE) << "Produced " << *samples_per_channel <<
167       " samples/channel for " << *num_channels << " channel(s)";
168   if (error != 0) {
169     LOG_FERR1(LS_WARNING, GetAudioInternal, error);
170     error_code_ = error;
171     return kFail;
172   }
173   if (type) {
174     *type = LastOutputType();
175   }
176   return kOK;
177 }
178 
RegisterPayloadType(enum NetEqDecoder codec,uint8_t rtp_payload_type)179 int NetEqImpl::RegisterPayloadType(enum NetEqDecoder codec,
180                                    uint8_t rtp_payload_type) {
181   CriticalSectionScoped lock(crit_sect_.get());
182   LOG_API2(static_cast<int>(rtp_payload_type), codec);
183   int ret = decoder_database_->RegisterPayload(rtp_payload_type, codec);
184   if (ret != DecoderDatabase::kOK) {
185     LOG_FERR2(LS_WARNING, RegisterPayload, static_cast<int>(rtp_payload_type),
186               codec);
187     switch (ret) {
188       case DecoderDatabase::kInvalidRtpPayloadType:
189         error_code_ = kInvalidRtpPayloadType;
190         break;
191       case DecoderDatabase::kCodecNotSupported:
192         error_code_ = kCodecNotSupported;
193         break;
194       case DecoderDatabase::kDecoderExists:
195         error_code_ = kDecoderExists;
196         break;
197       default:
198         error_code_ = kOtherError;
199     }
200     return kFail;
201   }
202   return kOK;
203 }
204 
RegisterExternalDecoder(AudioDecoder * decoder,enum NetEqDecoder codec,uint8_t rtp_payload_type)205 int NetEqImpl::RegisterExternalDecoder(AudioDecoder* decoder,
206                                        enum NetEqDecoder codec,
207                                        uint8_t rtp_payload_type) {
208   CriticalSectionScoped lock(crit_sect_.get());
209   LOG_API2(static_cast<int>(rtp_payload_type), codec);
210   if (!decoder) {
211     LOG(LS_ERROR) << "Cannot register external decoder with NULL pointer";
212     assert(false);
213     return kFail;
214   }
215   const int sample_rate_hz = CodecSampleRateHz(codec);
216   int ret = decoder_database_->InsertExternal(rtp_payload_type, codec,
217                                               sample_rate_hz, decoder);
218   if (ret != DecoderDatabase::kOK) {
219     LOG_FERR2(LS_WARNING, InsertExternal, static_cast<int>(rtp_payload_type),
220               codec);
221     switch (ret) {
222       case DecoderDatabase::kInvalidRtpPayloadType:
223         error_code_ = kInvalidRtpPayloadType;
224         break;
225       case DecoderDatabase::kCodecNotSupported:
226         error_code_ = kCodecNotSupported;
227         break;
228       case DecoderDatabase::kDecoderExists:
229         error_code_ = kDecoderExists;
230         break;
231       case DecoderDatabase::kInvalidSampleRate:
232         error_code_ = kInvalidSampleRate;
233         break;
234       case DecoderDatabase::kInvalidPointer:
235         error_code_ = kInvalidPointer;
236         break;
237       default:
238         error_code_ = kOtherError;
239     }
240     return kFail;
241   }
242   return kOK;
243 }
244 
RemovePayloadType(uint8_t rtp_payload_type)245 int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) {
246   CriticalSectionScoped lock(crit_sect_.get());
247   LOG_API1(static_cast<int>(rtp_payload_type));
248   int ret = decoder_database_->Remove(rtp_payload_type);
249   if (ret == DecoderDatabase::kOK) {
250     return kOK;
251   } else if (ret == DecoderDatabase::kDecoderNotFound) {
252     error_code_ = kDecoderNotFound;
253   } else {
254     error_code_ = kOtherError;
255   }
256   LOG_FERR1(LS_WARNING, Remove, static_cast<int>(rtp_payload_type));
257   return kFail;
258 }
259 
SetMinimumDelay(int delay_ms)260 bool NetEqImpl::SetMinimumDelay(int delay_ms) {
261   CriticalSectionScoped lock(crit_sect_.get());
262   if (delay_ms >= 0 && delay_ms < 10000) {
263     assert(delay_manager_.get());
264     return delay_manager_->SetMinimumDelay(delay_ms);
265   }
266   return false;
267 }
268 
SetMaximumDelay(int delay_ms)269 bool NetEqImpl::SetMaximumDelay(int delay_ms) {
270   CriticalSectionScoped lock(crit_sect_.get());
271   if (delay_ms >= 0 && delay_ms < 10000) {
272     assert(delay_manager_.get());
273     return delay_manager_->SetMaximumDelay(delay_ms);
274   }
275   return false;
276 }
277 
LeastRequiredDelayMs() const278 int NetEqImpl::LeastRequiredDelayMs() const {
279   CriticalSectionScoped lock(crit_sect_.get());
280   assert(delay_manager_.get());
281   return delay_manager_->least_required_delay_ms();
282 }
283 
284 // Deprecated.
285 // TODO(henrik.lundin) Delete.
SetPlayoutMode(NetEqPlayoutMode mode)286 void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) {
287   CriticalSectionScoped lock(crit_sect_.get());
288   if (mode != playout_mode_) {
289     playout_mode_ = mode;
290     CreateDecisionLogic();
291   }
292 }
293 
294 // Deprecated.
295 // TODO(henrik.lundin) Delete.
PlayoutMode() const296 NetEqPlayoutMode NetEqImpl::PlayoutMode() const {
297   CriticalSectionScoped lock(crit_sect_.get());
298   return playout_mode_;
299 }
300 
NetworkStatistics(NetEqNetworkStatistics * stats)301 int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) {
302   CriticalSectionScoped lock(crit_sect_.get());
303   assert(decoder_database_.get());
304   const int total_samples_in_buffers = packet_buffer_->NumSamplesInBuffer(
305       decoder_database_.get(), decoder_frame_length_) +
306           static_cast<int>(sync_buffer_->FutureLength());
307   assert(delay_manager_.get());
308   assert(decision_logic_.get());
309   stats_.GetNetworkStatistics(fs_hz_, total_samples_in_buffers,
310                               decoder_frame_length_, *delay_manager_.get(),
311                               *decision_logic_.get(), stats);
312   return 0;
313 }
314 
WaitingTimes(std::vector<int> * waiting_times)315 void NetEqImpl::WaitingTimes(std::vector<int>* waiting_times) {
316   CriticalSectionScoped lock(crit_sect_.get());
317   stats_.WaitingTimes(waiting_times);
318 }
319 
GetRtcpStatistics(RtcpStatistics * stats)320 void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) {
321   CriticalSectionScoped lock(crit_sect_.get());
322   if (stats) {
323     rtcp_.GetStatistics(false, stats);
324   }
325 }
326 
GetRtcpStatisticsNoReset(RtcpStatistics * stats)327 void NetEqImpl::GetRtcpStatisticsNoReset(RtcpStatistics* stats) {
328   CriticalSectionScoped lock(crit_sect_.get());
329   if (stats) {
330     rtcp_.GetStatistics(true, stats);
331   }
332 }
333 
EnableVad()334 void NetEqImpl::EnableVad() {
335   CriticalSectionScoped lock(crit_sect_.get());
336   assert(vad_.get());
337   vad_->Enable();
338 }
339 
DisableVad()340 void NetEqImpl::DisableVad() {
341   CriticalSectionScoped lock(crit_sect_.get());
342   assert(vad_.get());
343   vad_->Disable();
344 }
345 
GetPlayoutTimestamp(uint32_t * timestamp)346 bool NetEqImpl::GetPlayoutTimestamp(uint32_t* timestamp) {
347   CriticalSectionScoped lock(crit_sect_.get());
348   if (first_packet_) {
349     // We don't have a valid RTP timestamp until we have decoded our first
350     // RTP packet.
351     return false;
352   }
353   *timestamp = timestamp_scaler_->ToExternal(playout_timestamp_);
354   return true;
355 }
356 
LastError() const357 int NetEqImpl::LastError() const {
358   CriticalSectionScoped lock(crit_sect_.get());
359   return error_code_;
360 }
361 
LastDecoderError()362 int NetEqImpl::LastDecoderError() {
363   CriticalSectionScoped lock(crit_sect_.get());
364   return decoder_error_code_;
365 }
366 
FlushBuffers()367 void NetEqImpl::FlushBuffers() {
368   CriticalSectionScoped lock(crit_sect_.get());
369   LOG_API0();
370   packet_buffer_->Flush();
371   assert(sync_buffer_.get());
372   assert(expand_.get());
373   sync_buffer_->Flush();
374   sync_buffer_->set_next_index(sync_buffer_->next_index() -
375                                expand_->overlap_length());
376   // Set to wait for new codec.
377   first_packet_ = true;
378 }
379 
PacketBufferStatistics(int * current_num_packets,int * max_num_packets) const380 void NetEqImpl::PacketBufferStatistics(int* current_num_packets,
381                                        int* max_num_packets) const {
382   CriticalSectionScoped lock(crit_sect_.get());
383   packet_buffer_->BufferStat(current_num_packets, max_num_packets);
384 }
385 
DecodedRtpInfo(int * sequence_number,uint32_t * timestamp) const386 int NetEqImpl::DecodedRtpInfo(int* sequence_number, uint32_t* timestamp) const {
387   CriticalSectionScoped lock(crit_sect_.get());
388   if (decoded_packet_sequence_number_ < 0)
389     return -1;
390   *sequence_number = decoded_packet_sequence_number_;
391   *timestamp = decoded_packet_timestamp_;
392   return 0;
393 }
394 
sync_buffer_for_test() const395 const SyncBuffer* NetEqImpl::sync_buffer_for_test() const {
396   CriticalSectionScoped lock(crit_sect_.get());
397   return sync_buffer_.get();
398 }
399 
400 // Methods below this line are private.
401 
InsertPacketInternal(const WebRtcRTPHeader & rtp_header,const uint8_t * payload,size_t length_bytes,uint32_t receive_timestamp,bool is_sync_packet)402 int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
403                                     const uint8_t* payload,
404                                     size_t length_bytes,
405                                     uint32_t receive_timestamp,
406                                     bool is_sync_packet) {
407   if (!payload) {
408     LOG_F(LS_ERROR) << "payload == NULL";
409     return kInvalidPointer;
410   }
411   // Sanity checks for sync-packets.
412   if (is_sync_packet) {
413     if (decoder_database_->IsDtmf(rtp_header.header.payloadType) ||
414         decoder_database_->IsRed(rtp_header.header.payloadType) ||
415         decoder_database_->IsComfortNoise(rtp_header.header.payloadType)) {
416       LOG_F(LS_ERROR) << "Sync-packet with an unacceptable payload type "
417                       << static_cast<int>(rtp_header.header.payloadType);
418       return kSyncPacketNotAccepted;
419     }
420     if (first_packet_ ||
421         rtp_header.header.payloadType != current_rtp_payload_type_ ||
422         rtp_header.header.ssrc != ssrc_) {
423       // Even if |current_rtp_payload_type_| is 0xFF, sync-packet isn't
424       // accepted.
425       LOG_F(LS_ERROR)
426           << "Changing codec, SSRC or first packet with sync-packet.";
427       return kSyncPacketNotAccepted;
428     }
429   }
430   PacketList packet_list;
431   RTPHeader main_header;
432   {
433     // Convert to Packet.
434     // Create |packet| within this separate scope, since it should not be used
435     // directly once it's been inserted in the packet list. This way, |packet|
436     // is not defined outside of this block.
437     Packet* packet = new Packet;
438     packet->header.markerBit = false;
439     packet->header.payloadType = rtp_header.header.payloadType;
440     packet->header.sequenceNumber = rtp_header.header.sequenceNumber;
441     packet->header.timestamp = rtp_header.header.timestamp;
442     packet->header.ssrc = rtp_header.header.ssrc;
443     packet->header.numCSRCs = 0;
444     packet->payload_length = length_bytes;
445     packet->primary = true;
446     packet->waiting_time = 0;
447     packet->payload = new uint8_t[packet->payload_length];
448     packet->sync_packet = is_sync_packet;
449     if (!packet->payload) {
450       LOG_F(LS_ERROR) << "Payload pointer is NULL.";
451     }
452     assert(payload);  // Already checked above.
453     memcpy(packet->payload, payload, packet->payload_length);
454     // Insert packet in a packet list.
455     packet_list.push_back(packet);
456     // Save main payloads header for later.
457     memcpy(&main_header, &packet->header, sizeof(main_header));
458   }
459 
460   bool update_sample_rate_and_channels = false;
461   // Reinitialize NetEq if it's needed (changed SSRC or first call).
462   if ((main_header.ssrc != ssrc_) || first_packet_) {
463     // Note: |first_packet_| will be cleared further down in this method, once
464     // the packet has been successfully inserted into the packet buffer.
465 
466     rtcp_.Init(main_header.sequenceNumber);
467 
468     // Flush the packet buffer and DTMF buffer.
469     packet_buffer_->Flush();
470     dtmf_buffer_->Flush();
471 
472     // Store new SSRC.
473     ssrc_ = main_header.ssrc;
474 
475     // Update audio buffer timestamp.
476     sync_buffer_->IncreaseEndTimestamp(main_header.timestamp - timestamp_);
477 
478     // Update codecs.
479     timestamp_ = main_header.timestamp;
480     current_rtp_payload_type_ = main_header.payloadType;
481 
482     // Reset timestamp scaling.
483     timestamp_scaler_->Reset();
484 
485     // Trigger an update of sampling rate and the number of channels.
486     update_sample_rate_and_channels = true;
487   }
488 
489   // Update RTCP statistics, only for regular packets.
490   if (!is_sync_packet)
491     rtcp_.Update(main_header, receive_timestamp);
492 
493   // Check for RED payload type, and separate payloads into several packets.
494   if (decoder_database_->IsRed(main_header.payloadType)) {
495     assert(!is_sync_packet);  // We had a sanity check for this.
496     if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
497       LOG_FERR1(LS_WARNING, SplitRed, packet_list.size());
498       PacketBuffer::DeleteAllPackets(&packet_list);
499       return kRedundancySplitError;
500     }
501     // Only accept a few RED payloads of the same type as the main data,
502     // DTMF events and CNG.
503     payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
504     // Update the stored main payload header since the main payload has now
505     // changed.
506     memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
507   }
508 
509   // Check payload types.
510   if (decoder_database_->CheckPayloadTypes(packet_list) ==
511       DecoderDatabase::kDecoderNotFound) {
512     LOG_FERR1(LS_WARNING, CheckPayloadTypes, packet_list.size());
513     PacketBuffer::DeleteAllPackets(&packet_list);
514     return kUnknownRtpPayloadType;
515   }
516 
517   // Scale timestamp to internal domain (only for some codecs).
518   timestamp_scaler_->ToInternal(&packet_list);
519 
520   // Process DTMF payloads. Cycle through the list of packets, and pick out any
521   // DTMF payloads found.
522   PacketList::iterator it = packet_list.begin();
523   while (it != packet_list.end()) {
524     Packet* current_packet = (*it);
525     assert(current_packet);
526     assert(current_packet->payload);
527     if (decoder_database_->IsDtmf(current_packet->header.payloadType)) {
528       assert(!current_packet->sync_packet);  // We had a sanity check for this.
529       DtmfEvent event;
530       int ret = DtmfBuffer::ParseEvent(
531           current_packet->header.timestamp,
532           current_packet->payload,
533           current_packet->payload_length,
534           &event);
535       if (ret != DtmfBuffer::kOK) {
536         LOG_FERR2(LS_WARNING, ParseEvent, ret,
537                   current_packet->payload_length);
538         PacketBuffer::DeleteAllPackets(&packet_list);
539         return kDtmfParsingError;
540       }
541       if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) {
542         LOG_FERR0(LS_WARNING, InsertEvent);
543         PacketBuffer::DeleteAllPackets(&packet_list);
544         return kDtmfInsertError;
545       }
546       // TODO(hlundin): Let the destructor of Packet handle the payload.
547       delete [] current_packet->payload;
548       delete current_packet;
549       it = packet_list.erase(it);
550     } else {
551       ++it;
552     }
553   }
554 
555   // Check for FEC in packets, and separate payloads into several packets.
556   int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get());
557   if (ret != PayloadSplitter::kOK) {
558     LOG_FERR1(LS_WARNING, SplitFec, packet_list.size());
559     PacketBuffer::DeleteAllPackets(&packet_list);
560     switch (ret) {
561       case PayloadSplitter::kUnknownPayloadType:
562         return kUnknownRtpPayloadType;
563       default:
564         return kOtherError;
565     }
566   }
567 
568   // Split payloads into smaller chunks. This also verifies that all payloads
569   // are of a known payload type. SplitAudio() method is protected against
570   // sync-packets.
571   ret = payload_splitter_->SplitAudio(&packet_list, *decoder_database_);
572   if (ret != PayloadSplitter::kOK) {
573     LOG_FERR1(LS_WARNING, SplitAudio, packet_list.size());
574     PacketBuffer::DeleteAllPackets(&packet_list);
575     switch (ret) {
576       case PayloadSplitter::kUnknownPayloadType:
577         return kUnknownRtpPayloadType;
578       case PayloadSplitter::kFrameSplitError:
579         return kFrameSplitError;
580       default:
581         return kOtherError;
582     }
583   }
584 
585   // Update bandwidth estimate, if the packet is not sync-packet.
586   if (!packet_list.empty() && !packet_list.front()->sync_packet) {
587     // The list can be empty here if we got nothing but DTMF payloads.
588     AudioDecoder* decoder =
589         decoder_database_->GetDecoder(main_header.payloadType);
590     assert(decoder);  // Should always get a valid object, since we have
591                       // already checked that the payload types are known.
592     decoder->IncomingPacket(packet_list.front()->payload,
593                             packet_list.front()->payload_length,
594                             packet_list.front()->header.sequenceNumber,
595                             packet_list.front()->header.timestamp,
596                             receive_timestamp);
597   }
598 
599   // Insert packets in buffer.
600   int temp_bufsize = packet_buffer_->NumPacketsInBuffer();
601   ret = packet_buffer_->InsertPacketList(
602       &packet_list,
603       *decoder_database_,
604       &current_rtp_payload_type_,
605       &current_cng_rtp_payload_type_);
606   if (ret == PacketBuffer::kFlushed) {
607     // Reset DSP timestamp etc. if packet buffer flushed.
608     new_codec_ = true;
609     update_sample_rate_and_channels = true;
610     LOG_F(LS_WARNING) << "Packet buffer flushed";
611   } else if (ret != PacketBuffer::kOK) {
612     LOG_FERR1(LS_WARNING, InsertPacketList, packet_list.size());
613     PacketBuffer::DeleteAllPackets(&packet_list);
614     return kOtherError;
615   }
616 
617   if (first_packet_) {
618     first_packet_ = false;
619     // Update the codec on the next GetAudio call.
620     new_codec_ = true;
621   }
622 
623   if (current_rtp_payload_type_ != 0xFF) {
624     const DecoderDatabase::DecoderInfo* dec_info =
625         decoder_database_->GetDecoderInfo(current_rtp_payload_type_);
626     if (!dec_info) {
627       assert(false);  // Already checked that the payload type is known.
628     }
629   }
630 
631   if (update_sample_rate_and_channels && !packet_buffer_->Empty()) {
632     // We do not use |current_rtp_payload_type_| to |set payload_type|, but
633     // get the next RTP header from |packet_buffer_| to obtain the payload type.
634     // The reason for it is the following corner case. If NetEq receives a
635     // CNG packet with a sample rate different than the current CNG then it
636     // flushes its buffer, assuming send codec must have been changed. However,
637     // payload type of the hypothetically new send codec is not known.
638     const RTPHeader* rtp_header = packet_buffer_->NextRtpHeader();
639     assert(rtp_header);
640     int payload_type = rtp_header->payloadType;
641     AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type);
642     assert(decoder);  // Payloads are already checked to be valid.
643     const DecoderDatabase::DecoderInfo* decoder_info =
644         decoder_database_->GetDecoderInfo(payload_type);
645     assert(decoder_info);
646     if (decoder_info->fs_hz != fs_hz_ ||
647         decoder->Channels() != algorithm_buffer_->Channels())
648       SetSampleRateAndChannels(decoder_info->fs_hz, decoder->Channels());
649   }
650 
651   // TODO(hlundin): Move this code to DelayManager class.
652   const DecoderDatabase::DecoderInfo* dec_info =
653           decoder_database_->GetDecoderInfo(main_header.payloadType);
654   assert(dec_info);  // Already checked that the payload type is known.
655   delay_manager_->LastDecoderType(dec_info->codec_type);
656   if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
657     // Calculate the total speech length carried in each packet.
658     temp_bufsize = packet_buffer_->NumPacketsInBuffer() - temp_bufsize;
659     temp_bufsize *= decoder_frame_length_;
660 
661     if ((temp_bufsize > 0) &&
662         (temp_bufsize != decision_logic_->packet_length_samples())) {
663       decision_logic_->set_packet_length_samples(temp_bufsize);
664       delay_manager_->SetPacketAudioLength((1000 * temp_bufsize) / fs_hz_);
665     }
666 
667     // Update statistics.
668     if ((int32_t) (main_header.timestamp - timestamp_) >= 0 &&
669         !new_codec_) {
670       // Only update statistics if incoming packet is not older than last played
671       // out packet, and if new codec flag is not set.
672       delay_manager_->Update(main_header.sequenceNumber, main_header.timestamp,
673                              fs_hz_);
674     }
675   } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
676     // This is first "normal" packet after CNG or DTMF.
677     // Reset packet time counter and measure time until next packet,
678     // but don't update statistics.
679     delay_manager_->set_last_pack_cng_or_dtmf(0);
680     delay_manager_->ResetPacketIatCount();
681   }
682   return 0;
683 }
684 
GetAudioInternal(size_t max_length,int16_t * output,int * samples_per_channel,int * num_channels)685 int NetEqImpl::GetAudioInternal(size_t max_length, int16_t* output,
686                                 int* samples_per_channel, int* num_channels) {
687   PacketList packet_list;
688   DtmfEvent dtmf_event;
689   Operations operation;
690   bool play_dtmf;
691   int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
692                                  &play_dtmf);
693   if (return_value != 0) {
694     LOG_FERR1(LS_WARNING, GetDecision, return_value);
695     assert(false);
696     last_mode_ = kModeError;
697     return return_value;
698   }
699   LOG(LS_VERBOSE) << "GetDecision returned operation=" << operation <<
700       " and " << packet_list.size() << " packet(s)";
701 
702   AudioDecoder::SpeechType speech_type;
703   int length = 0;
704   int decode_return_value = Decode(&packet_list, &operation,
705                                    &length, &speech_type);
706 
707   assert(vad_.get());
708   bool sid_frame_available =
709       (operation == kRfc3389Cng && !packet_list.empty());
710   vad_->Update(decoded_buffer_.get(), length, speech_type,
711                sid_frame_available, fs_hz_);
712 
713   algorithm_buffer_->Clear();
714   switch (operation) {
715     case kNormal: {
716       DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
717       break;
718     }
719     case kMerge: {
720       DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
721       break;
722     }
723     case kExpand: {
724       return_value = DoExpand(play_dtmf);
725       break;
726     }
727     case kAccelerate: {
728       return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type,
729                                   play_dtmf);
730       break;
731     }
732     case kPreemptiveExpand: {
733       return_value = DoPreemptiveExpand(decoded_buffer_.get(), length,
734                                         speech_type, play_dtmf);
735       break;
736     }
737     case kRfc3389Cng:
738     case kRfc3389CngNoPacket: {
739       return_value = DoRfc3389Cng(&packet_list, play_dtmf);
740       break;
741     }
742     case kCodecInternalCng: {
743       // This handles the case when there is no transmission and the decoder
744       // should produce internal comfort noise.
745       // TODO(hlundin): Write test for codec-internal CNG.
746       DoCodecInternalCng();
747       break;
748     }
749     case kDtmf: {
750       // TODO(hlundin): Write test for this.
751       return_value = DoDtmf(dtmf_event, &play_dtmf);
752       break;
753     }
754     case kAlternativePlc: {
755       // TODO(hlundin): Write test for this.
756       DoAlternativePlc(false);
757       break;
758     }
759     case kAlternativePlcIncreaseTimestamp: {
760       // TODO(hlundin): Write test for this.
761       DoAlternativePlc(true);
762       break;
763     }
764     case kAudioRepetitionIncreaseTimestamp: {
765       // TODO(hlundin): Write test for this.
766       sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
767       // Skipping break on purpose. Execution should move on into the
768       // next case.
769       FALLTHROUGH();
770     }
771     case kAudioRepetition: {
772       // TODO(hlundin): Write test for this.
773       // Copy last |output_size_samples_| from |sync_buffer_| to
774       // |algorithm_buffer|.
775       algorithm_buffer_->PushBackFromIndex(
776           *sync_buffer_, sync_buffer_->Size() - output_size_samples_);
777       expand_->Reset();
778       break;
779     }
780     case kUndefined: {
781       LOG_F(LS_ERROR) << "Invalid operation kUndefined.";
782       assert(false);  // This should not happen.
783       last_mode_ = kModeError;
784       return kInvalidOperation;
785     }
786   }  // End of switch.
787   if (return_value < 0) {
788     return return_value;
789   }
790 
791   if (last_mode_ != kModeRfc3389Cng) {
792     comfort_noise_->Reset();
793   }
794 
795   // Copy from |algorithm_buffer| to |sync_buffer_|.
796   sync_buffer_->PushBack(*algorithm_buffer_);
797 
798   // Extract data from |sync_buffer_| to |output|.
799   size_t num_output_samples_per_channel = output_size_samples_;
800   size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels();
801   if (num_output_samples > max_length) {
802     LOG(LS_WARNING) << "Output array is too short. " << max_length << " < " <<
803         output_size_samples_ << " * " << sync_buffer_->Channels();
804     num_output_samples = max_length;
805     num_output_samples_per_channel = static_cast<int>(
806         max_length / sync_buffer_->Channels());
807   }
808   int samples_from_sync = static_cast<int>(
809       sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
810                                             output));
811   *num_channels = static_cast<int>(sync_buffer_->Channels());
812   LOG(LS_VERBOSE) << "Sync buffer (" << *num_channels << " channel(s)):" <<
813       " insert " << algorithm_buffer_->Size() << " samples, extract " <<
814       samples_from_sync << " samples";
815   if (samples_from_sync != output_size_samples_) {
816     LOG_F(LS_ERROR) << "samples_from_sync != output_size_samples_";
817     // TODO(minyue): treatment of under-run, filling zeros
818     memset(output, 0, num_output_samples * sizeof(int16_t));
819     *samples_per_channel = output_size_samples_;
820     return kSampleUnderrun;
821   }
822   *samples_per_channel = output_size_samples_;
823 
824   // Should always have overlap samples left in the |sync_buffer_|.
825   assert(sync_buffer_->FutureLength() >= expand_->overlap_length());
826 
827   if (play_dtmf) {
828     return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(), output);
829   }
830 
831   // Update the background noise parameters if last operation wrote data
832   // straight from the decoder to the |sync_buffer_|. That is, none of the
833   // operations that modify the signal can be followed by a parameter update.
834   if ((last_mode_ == kModeNormal) ||
835       (last_mode_ == kModeAccelerateFail) ||
836       (last_mode_ == kModePreemptiveExpandFail) ||
837       (last_mode_ == kModeRfc3389Cng) ||
838       (last_mode_ == kModeCodecInternalCng)) {
839     background_noise_->Update(*sync_buffer_, *vad_.get());
840   }
841 
842   if (operation == kDtmf) {
843     // DTMF data was written the end of |sync_buffer_|.
844     // Update index to end of DTMF data in |sync_buffer_|.
845     sync_buffer_->set_dtmf_index(sync_buffer_->Size());
846   }
847 
848   if (last_mode_ != kModeExpand) {
849     // If last operation was not expand, calculate the |playout_timestamp_| from
850     // the |sync_buffer_|. However, do not update the |playout_timestamp_| if it
851     // would be moved "backwards".
852     uint32_t temp_timestamp = sync_buffer_->end_timestamp() -
853         static_cast<uint32_t>(sync_buffer_->FutureLength());
854     if (static_cast<int32_t>(temp_timestamp - playout_timestamp_) > 0) {
855       playout_timestamp_ = temp_timestamp;
856     }
857   } else {
858     // Use dead reckoning to estimate the |playout_timestamp_|.
859     playout_timestamp_ += output_size_samples_;
860   }
861 
862   if (decode_return_value) return decode_return_value;
863   return return_value;
864 }
865 
GetDecision(Operations * operation,PacketList * packet_list,DtmfEvent * dtmf_event,bool * play_dtmf)866 int NetEqImpl::GetDecision(Operations* operation,
867                            PacketList* packet_list,
868                            DtmfEvent* dtmf_event,
869                            bool* play_dtmf) {
870   // Initialize output variables.
871   *play_dtmf = false;
872   *operation = kUndefined;
873 
874   // Increment time counters.
875   packet_buffer_->IncrementWaitingTimes();
876   stats_.IncreaseCounter(output_size_samples_, fs_hz_);
877 
878   assert(sync_buffer_.get());
879   uint32_t end_timestamp = sync_buffer_->end_timestamp();
880   if (!new_codec_) {
881     const uint32_t five_seconds_samples = 5 * fs_hz_;
882     packet_buffer_->DiscardOldPackets(end_timestamp, five_seconds_samples);
883   }
884   const RTPHeader* header = packet_buffer_->NextRtpHeader();
885 
886   if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
887     // Because of timestamp peculiarities, we have to "manually" disallow using
888     // a CNG packet with the same timestamp as the one that was last played.
889     // This can happen when using redundancy and will cause the timing to shift.
890     while (header && decoder_database_->IsComfortNoise(header->payloadType) &&
891            (end_timestamp >= header->timestamp ||
892             end_timestamp + decision_logic_->generated_noise_samples() >
893                 header->timestamp)) {
894       // Don't use this packet, discard it.
895       if (packet_buffer_->DiscardNextPacket() != PacketBuffer::kOK) {
896         assert(false);  // Must be ok by design.
897       }
898       // Check buffer again.
899       if (!new_codec_) {
900         packet_buffer_->DiscardOldPackets(end_timestamp, 5 * fs_hz_);
901       }
902       header = packet_buffer_->NextRtpHeader();
903     }
904   }
905 
906   assert(expand_.get());
907   const int samples_left = static_cast<int>(sync_buffer_->FutureLength() -
908       expand_->overlap_length());
909   if (last_mode_ == kModeAccelerateSuccess ||
910       last_mode_ == kModeAccelerateLowEnergy ||
911       last_mode_ == kModePreemptiveExpandSuccess ||
912       last_mode_ == kModePreemptiveExpandLowEnergy) {
913     // Subtract (samples_left + output_size_samples_) from sampleMemory.
914     decision_logic_->AddSampleMemory(-(samples_left + output_size_samples_));
915   }
916 
917   // Check if it is time to play a DTMF event.
918   if (dtmf_buffer_->GetEvent(end_timestamp +
919                              decision_logic_->generated_noise_samples(),
920                              dtmf_event)) {
921     *play_dtmf = true;
922   }
923 
924   // Get instruction.
925   assert(sync_buffer_.get());
926   assert(expand_.get());
927   *operation = decision_logic_->GetDecision(*sync_buffer_,
928                                             *expand_,
929                                             decoder_frame_length_,
930                                             header,
931                                             last_mode_,
932                                             *play_dtmf,
933                                             &reset_decoder_);
934 
935   // Check if we already have enough samples in the |sync_buffer_|. If so,
936   // change decision to normal, unless the decision was merge, accelerate, or
937   // preemptive expand.
938   if (samples_left >= output_size_samples_ &&
939       *operation != kMerge &&
940       *operation != kAccelerate &&
941       *operation != kPreemptiveExpand) {
942     *operation = kNormal;
943     return 0;
944   }
945 
946   decision_logic_->ExpandDecision(*operation);
947 
948   // Check conditions for reset.
949   if (new_codec_ || *operation == kUndefined) {
950     // The only valid reason to get kUndefined is that new_codec_ is set.
951     assert(new_codec_);
952     if (*play_dtmf && !header) {
953       timestamp_ = dtmf_event->timestamp;
954     } else {
955       assert(header);
956       if (!header) {
957         LOG_F(LS_ERROR) << "Packet missing where it shouldn't.";
958         return -1;
959       }
960       timestamp_ = header->timestamp;
961       if (*operation == kRfc3389CngNoPacket
962 #ifndef LEGACY_BITEXACT
963           // Without this check, it can happen that a non-CNG packet is sent to
964           // the CNG decoder as if it was a SID frame. This is clearly a bug,
965           // but is kept for now to maintain bit-exactness with the test
966           // vectors.
967           && decoder_database_->IsComfortNoise(header->payloadType)
968 #endif
969       ) {
970         // Change decision to CNG packet, since we do have a CNG packet, but it
971         // was considered too early to use. Now, use it anyway.
972         *operation = kRfc3389Cng;
973       } else if (*operation != kRfc3389Cng) {
974         *operation = kNormal;
975       }
976     }
977     // Adjust |sync_buffer_| timestamp before setting |end_timestamp| to the
978     // new value.
979     sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp);
980     end_timestamp = timestamp_;
981     new_codec_ = false;
982     decision_logic_->SoftReset();
983     buffer_level_filter_->Reset();
984     delay_manager_->Reset();
985     stats_.ResetMcu();
986   }
987 
988   int required_samples = output_size_samples_;
989   const int samples_10_ms = 80 * fs_mult_;
990   const int samples_20_ms = 2 * samples_10_ms;
991   const int samples_30_ms = 3 * samples_10_ms;
992 
993   switch (*operation) {
994     case kExpand: {
995       timestamp_ = end_timestamp;
996       return 0;
997     }
998     case kRfc3389CngNoPacket:
999     case kCodecInternalCng: {
1000       return 0;
1001     }
1002     case kDtmf: {
1003       // TODO(hlundin): Write test for this.
1004       // Update timestamp.
1005       timestamp_ = end_timestamp;
1006       if (decision_logic_->generated_noise_samples() > 0 &&
1007           last_mode_ != kModeDtmf) {
1008         // Make a jump in timestamp due to the recently played comfort noise.
1009         uint32_t timestamp_jump = decision_logic_->generated_noise_samples();
1010         sync_buffer_->IncreaseEndTimestamp(timestamp_jump);
1011         timestamp_ += timestamp_jump;
1012       }
1013       decision_logic_->set_generated_noise_samples(0);
1014       return 0;
1015     }
1016     case kAccelerate: {
1017       // In order to do a accelerate we need at least 30 ms of audio data.
1018       if (samples_left >= samples_30_ms) {
1019         // Already have enough data, so we do not need to extract any more.
1020         decision_logic_->set_sample_memory(samples_left);
1021         decision_logic_->set_prev_time_scale(true);
1022         return 0;
1023       } else if (samples_left >= samples_10_ms &&
1024           decoder_frame_length_ >= samples_30_ms) {
1025         // Avoid decoding more data as it might overflow the playout buffer.
1026         *operation = kNormal;
1027         return 0;
1028       } else if (samples_left < samples_20_ms &&
1029           decoder_frame_length_ < samples_30_ms) {
1030         // Build up decoded data by decoding at least 20 ms of audio data. Do
1031         // not perform accelerate yet, but wait until we only need to do one
1032         // decoding.
1033         required_samples = 2 * output_size_samples_;
1034         *operation = kNormal;
1035       }
1036       // If none of the above is true, we have one of two possible situations:
1037       // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or
1038       // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms.
1039       // In either case, we move on with the accelerate decision, and decode one
1040       // frame now.
1041       break;
1042     }
1043     case kPreemptiveExpand: {
1044       // In order to do a preemptive expand we need at least 30 ms of decoded
1045       // audio data.
1046       if ((samples_left >= samples_30_ms) ||
1047           (samples_left >= samples_10_ms &&
1048               decoder_frame_length_ >= samples_30_ms)) {
1049         // Already have enough data, so we do not need to extract any more.
1050         // Or, avoid decoding more data as it might overflow the playout buffer.
1051         // Still try preemptive expand, though.
1052         decision_logic_->set_sample_memory(samples_left);
1053         decision_logic_->set_prev_time_scale(true);
1054         return 0;
1055       }
1056       if (samples_left < samples_20_ms &&
1057           decoder_frame_length_ < samples_30_ms) {
1058         // Build up decoded data by decoding at least 20 ms of audio data.
1059         // Still try to perform preemptive expand.
1060         required_samples = 2 * output_size_samples_;
1061       }
1062       // Move on with the preemptive expand decision.
1063       break;
1064     }
1065     case kMerge: {
1066       required_samples =
1067           std::max(merge_->RequiredFutureSamples(), required_samples);
1068       break;
1069     }
1070     default: {
1071       // Do nothing.
1072     }
1073   }
1074 
1075   // Get packets from buffer.
1076   int extracted_samples = 0;
1077   if (header &&
1078       *operation != kAlternativePlc &&
1079       *operation != kAlternativePlcIncreaseTimestamp &&
1080       *operation != kAudioRepetition &&
1081       *operation != kAudioRepetitionIncreaseTimestamp) {
1082     sync_buffer_->IncreaseEndTimestamp(header->timestamp - end_timestamp);
1083     if (decision_logic_->CngOff()) {
1084       // Adjustment of timestamp only corresponds to an actual packet loss
1085       // if comfort noise is not played. If comfort noise was just played,
1086       // this adjustment of timestamp is only done to get back in sync with the
1087       // stream timestamp; no loss to report.
1088       stats_.LostSamples(header->timestamp - end_timestamp);
1089     }
1090 
1091     if (*operation != kRfc3389Cng) {
1092       // We are about to decode and use a non-CNG packet.
1093       decision_logic_->SetCngOff();
1094     }
1095     // Reset CNG timestamp as a new packet will be delivered.
1096     // (Also if this is a CNG packet, since playedOutTS is updated.)
1097     decision_logic_->set_generated_noise_samples(0);
1098 
1099     extracted_samples = ExtractPackets(required_samples, packet_list);
1100     if (extracted_samples < 0) {
1101       LOG_F(LS_WARNING) << "Failed to extract packets from buffer.";
1102       return kPacketBufferCorruption;
1103     }
1104   }
1105 
1106   if (*operation == kAccelerate ||
1107       *operation == kPreemptiveExpand) {
1108     decision_logic_->set_sample_memory(samples_left + extracted_samples);
1109     decision_logic_->set_prev_time_scale(true);
1110   }
1111 
1112   if (*operation == kAccelerate) {
1113     // Check that we have enough data (30ms) to do accelerate.
1114     if (extracted_samples + samples_left < samples_30_ms) {
1115       // TODO(hlundin): Write test for this.
1116       // Not enough, do normal operation instead.
1117       *operation = kNormal;
1118     }
1119   }
1120 
1121   timestamp_ = end_timestamp;
1122   return 0;
1123 }
1124 
Decode(PacketList * packet_list,Operations * operation,int * decoded_length,AudioDecoder::SpeechType * speech_type)1125 int NetEqImpl::Decode(PacketList* packet_list, Operations* operation,
1126                       int* decoded_length,
1127                       AudioDecoder::SpeechType* speech_type) {
1128   *speech_type = AudioDecoder::kSpeech;
1129   AudioDecoder* decoder = NULL;
1130   if (!packet_list->empty()) {
1131     const Packet* packet = packet_list->front();
1132     uint8_t payload_type = packet->header.payloadType;
1133     if (!decoder_database_->IsComfortNoise(payload_type)) {
1134       decoder = decoder_database_->GetDecoder(payload_type);
1135       assert(decoder);
1136       if (!decoder) {
1137         LOG_FERR1(LS_WARNING, GetDecoder, static_cast<int>(payload_type));
1138         PacketBuffer::DeleteAllPackets(packet_list);
1139         return kDecoderNotFound;
1140       }
1141       bool decoder_changed;
1142       decoder_database_->SetActiveDecoder(payload_type, &decoder_changed);
1143       if (decoder_changed) {
1144         // We have a new decoder. Re-init some values.
1145         const DecoderDatabase::DecoderInfo* decoder_info = decoder_database_
1146             ->GetDecoderInfo(payload_type);
1147         assert(decoder_info);
1148         if (!decoder_info) {
1149           LOG_FERR1(LS_WARNING, GetDecoderInfo, static_cast<int>(payload_type));
1150           PacketBuffer::DeleteAllPackets(packet_list);
1151           return kDecoderNotFound;
1152         }
1153         // If sampling rate or number of channels has changed, we need to make
1154         // a reset.
1155         if (decoder_info->fs_hz != fs_hz_ ||
1156             decoder->Channels() != algorithm_buffer_->Channels()) {
1157           // TODO(tlegrand): Add unittest to cover this event.
1158           SetSampleRateAndChannels(decoder_info->fs_hz, decoder->Channels());
1159         }
1160         sync_buffer_->set_end_timestamp(timestamp_);
1161         playout_timestamp_ = timestamp_;
1162       }
1163     }
1164   }
1165 
1166   if (reset_decoder_) {
1167     // TODO(hlundin): Write test for this.
1168     // Reset decoder.
1169     if (decoder) {
1170       decoder->Init();
1171     }
1172     // Reset comfort noise decoder.
1173     AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1174     if (cng_decoder) {
1175       cng_decoder->Init();
1176     }
1177     reset_decoder_ = false;
1178   }
1179 
1180 #ifdef LEGACY_BITEXACT
1181   // Due to a bug in old SignalMCU, it could happen that CNG operation was
1182   // decided, but a speech packet was provided. The speech packet will be used
1183   // to update the comfort noise decoder, as if it was a SID frame, which is
1184   // clearly wrong.
1185   if (*operation == kRfc3389Cng) {
1186     return 0;
1187   }
1188 #endif
1189 
1190   *decoded_length = 0;
1191   // Update codec-internal PLC state.
1192   if ((*operation == kMerge) && decoder && decoder->HasDecodePlc()) {
1193     decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]);
1194   }
1195 
1196   int return_value = DecodeLoop(packet_list, operation, decoder,
1197                                 decoded_length, speech_type);
1198 
1199   if (*decoded_length < 0) {
1200     // Error returned from the decoder.
1201     *decoded_length = 0;
1202     sync_buffer_->IncreaseEndTimestamp(decoder_frame_length_);
1203     int error_code = 0;
1204     if (decoder)
1205       error_code = decoder->ErrorCode();
1206     if (error_code != 0) {
1207       // Got some error code from the decoder.
1208       decoder_error_code_ = error_code;
1209       return_value = kDecoderErrorCode;
1210     } else {
1211       // Decoder does not implement error codes. Return generic error.
1212       return_value = kOtherDecoderError;
1213     }
1214     LOG_FERR2(LS_WARNING, DecodeLoop, error_code, packet_list->size());
1215     *operation = kExpand;  // Do expansion to get data instead.
1216   }
1217   if (*speech_type != AudioDecoder::kComfortNoise) {
1218     // Don't increment timestamp if codec returned CNG speech type
1219     // since in this case, the we will increment the CNGplayedTS counter.
1220     // Increase with number of samples per channel.
1221     assert(*decoded_length == 0 ||
1222            (decoder && decoder->Channels() == sync_buffer_->Channels()));
1223     sync_buffer_->IncreaseEndTimestamp(
1224         *decoded_length / static_cast<int>(sync_buffer_->Channels()));
1225   }
1226   return return_value;
1227 }
1228 
DecodeLoop(PacketList * packet_list,Operations * operation,AudioDecoder * decoder,int * decoded_length,AudioDecoder::SpeechType * speech_type)1229 int NetEqImpl::DecodeLoop(PacketList* packet_list, Operations* operation,
1230                           AudioDecoder* decoder, int* decoded_length,
1231                           AudioDecoder::SpeechType* speech_type) {
1232   Packet* packet = NULL;
1233   if (!packet_list->empty()) {
1234     packet = packet_list->front();
1235   }
1236   // Do decoding.
1237   while (packet &&
1238       !decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1239     assert(decoder);  // At this point, we must have a decoder object.
1240     // The number of channels in the |sync_buffer_| should be the same as the
1241     // number decoder channels.
1242     assert(sync_buffer_->Channels() == decoder->Channels());
1243     assert(decoded_buffer_length_ >= kMaxFrameSize * decoder->Channels());
1244     assert(*operation == kNormal || *operation == kAccelerate ||
1245            *operation == kMerge || *operation == kPreemptiveExpand);
1246     packet_list->pop_front();
1247     size_t payload_length = packet->payload_length;
1248     int16_t decode_length;
1249     if (packet->sync_packet) {
1250       // Decode to silence with the same frame size as the last decode.
1251       LOG(LS_VERBOSE) << "Decoding sync-packet: " <<
1252           " ts=" << packet->header.timestamp <<
1253           ", sn=" << packet->header.sequenceNumber <<
1254           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1255           ", ssrc=" << packet->header.ssrc <<
1256           ", len=" << packet->payload_length;
1257       memset(&decoded_buffer_[*decoded_length], 0,
1258              decoder_frame_length_ * decoder->Channels() *
1259                  sizeof(decoded_buffer_[0]));
1260       decode_length = decoder_frame_length_;
1261     } else if (!packet->primary) {
1262       // This is a redundant payload; call the special decoder method.
1263       LOG(LS_VERBOSE) << "Decoding packet (redundant):" <<
1264           " ts=" << packet->header.timestamp <<
1265           ", sn=" << packet->header.sequenceNumber <<
1266           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1267           ", ssrc=" << packet->header.ssrc <<
1268           ", len=" << packet->payload_length;
1269       decode_length = decoder->DecodeRedundant(
1270           packet->payload, packet->payload_length, fs_hz_,
1271           (decoded_buffer_length_ - *decoded_length) * sizeof(int16_t),
1272           &decoded_buffer_[*decoded_length], speech_type);
1273     } else {
1274       LOG(LS_VERBOSE) << "Decoding packet: ts=" << packet->header.timestamp <<
1275           ", sn=" << packet->header.sequenceNumber <<
1276           ", pt=" << static_cast<int>(packet->header.payloadType) <<
1277           ", ssrc=" << packet->header.ssrc <<
1278           ", len=" << packet->payload_length;
1279       decode_length =
1280           decoder->Decode(
1281               packet->payload, packet->payload_length, fs_hz_,
1282               (decoded_buffer_length_ - *decoded_length) * sizeof(int16_t),
1283               &decoded_buffer_[*decoded_length], speech_type);
1284     }
1285 
1286     delete[] packet->payload;
1287     delete packet;
1288     packet = NULL;
1289     if (decode_length > 0) {
1290       *decoded_length += decode_length;
1291       // Update |decoder_frame_length_| with number of samples per channel.
1292       decoder_frame_length_ =
1293           decode_length / static_cast<int>(decoder->Channels());
1294       LOG(LS_VERBOSE) << "Decoded " << decode_length << " samples ("
1295                       << decoder->Channels() << " channel(s) -> "
1296                       << decoder_frame_length_ << " samples per channel)";
1297     } else if (decode_length < 0) {
1298       // Error.
1299       LOG_FERR2(LS_WARNING, Decode, decode_length, payload_length);
1300       *decoded_length = -1;
1301       PacketBuffer::DeleteAllPackets(packet_list);
1302       break;
1303     }
1304     if (*decoded_length > static_cast<int>(decoded_buffer_length_)) {
1305       // Guard against overflow.
1306       LOG_F(LS_WARNING) << "Decoded too much.";
1307       PacketBuffer::DeleteAllPackets(packet_list);
1308       return kDecodedTooMuch;
1309     }
1310     if (!packet_list->empty()) {
1311       packet = packet_list->front();
1312     } else {
1313       packet = NULL;
1314     }
1315   }  // End of decode loop.
1316 
1317   // If the list is not empty at this point, either a decoding error terminated
1318   // the while-loop, or list must hold exactly one CNG packet.
1319   assert(packet_list->empty() || *decoded_length < 0 ||
1320          (packet_list->size() == 1 && packet &&
1321              decoder_database_->IsComfortNoise(packet->header.payloadType)));
1322   return 0;
1323 }
1324 
DoNormal(const int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1325 void NetEqImpl::DoNormal(const int16_t* decoded_buffer, size_t decoded_length,
1326                          AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1327   assert(normal_.get());
1328   assert(mute_factor_array_.get());
1329   normal_->Process(decoded_buffer, decoded_length, last_mode_,
1330                    mute_factor_array_.get(), algorithm_buffer_.get());
1331   if (decoded_length != 0) {
1332     last_mode_ = kModeNormal;
1333   }
1334 
1335   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1336   if ((speech_type == AudioDecoder::kComfortNoise)
1337       || ((last_mode_ == kModeCodecInternalCng)
1338           && (decoded_length == 0))) {
1339     // TODO(hlundin): Remove second part of || statement above.
1340     last_mode_ = kModeCodecInternalCng;
1341   }
1342 
1343   if (!play_dtmf) {
1344     dtmf_tone_generator_->Reset();
1345   }
1346 }
1347 
DoMerge(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1348 void NetEqImpl::DoMerge(int16_t* decoded_buffer, size_t decoded_length,
1349                         AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1350   assert(mute_factor_array_.get());
1351   assert(merge_.get());
1352   int new_length = merge_->Process(decoded_buffer, decoded_length,
1353                                    mute_factor_array_.get(),
1354                                    algorithm_buffer_.get());
1355   int expand_length_correction = new_length -
1356       static_cast<int>(decoded_length / algorithm_buffer_->Channels());
1357 
1358   // Update in-call and post-call statistics.
1359   if (expand_->MuteFactor(0) == 0) {
1360     // Expand generates only noise.
1361     stats_.ExpandedNoiseSamples(expand_length_correction);
1362   } else {
1363     // Expansion generates more than only noise.
1364     stats_.ExpandedVoiceSamples(expand_length_correction);
1365   }
1366 
1367   last_mode_ = kModeMerge;
1368   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1369   if (speech_type == AudioDecoder::kComfortNoise) {
1370     last_mode_ = kModeCodecInternalCng;
1371   }
1372   expand_->Reset();
1373   if (!play_dtmf) {
1374     dtmf_tone_generator_->Reset();
1375   }
1376 }
1377 
DoExpand(bool play_dtmf)1378 int NetEqImpl::DoExpand(bool play_dtmf) {
1379   while ((sync_buffer_->FutureLength() - expand_->overlap_length()) <
1380       static_cast<size_t>(output_size_samples_)) {
1381     algorithm_buffer_->Clear();
1382     int return_value = expand_->Process(algorithm_buffer_.get());
1383     int length = static_cast<int>(algorithm_buffer_->Size());
1384 
1385     // Update in-call and post-call statistics.
1386     if (expand_->MuteFactor(0) == 0) {
1387       // Expand operation generates only noise.
1388       stats_.ExpandedNoiseSamples(length);
1389     } else {
1390       // Expand operation generates more than only noise.
1391       stats_.ExpandedVoiceSamples(length);
1392     }
1393 
1394     last_mode_ = kModeExpand;
1395 
1396     if (return_value < 0) {
1397       return return_value;
1398     }
1399 
1400     sync_buffer_->PushBack(*algorithm_buffer_);
1401     algorithm_buffer_->Clear();
1402   }
1403   if (!play_dtmf) {
1404     dtmf_tone_generator_->Reset();
1405   }
1406   return 0;
1407 }
1408 
DoAccelerate(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1409 int NetEqImpl::DoAccelerate(int16_t* decoded_buffer, size_t decoded_length,
1410                             AudioDecoder::SpeechType speech_type,
1411                             bool play_dtmf) {
1412   const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1413   size_t borrowed_samples_per_channel = 0;
1414   size_t num_channels = algorithm_buffer_->Channels();
1415   size_t decoded_length_per_channel = decoded_length / num_channels;
1416   if (decoded_length_per_channel < required_samples) {
1417     // Must move data from the |sync_buffer_| in order to get 30 ms.
1418     borrowed_samples_per_channel = static_cast<int>(required_samples -
1419         decoded_length_per_channel);
1420     memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1421             decoded_buffer,
1422             sizeof(int16_t) * decoded_length);
1423     sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1424                                          decoded_buffer);
1425     decoded_length = required_samples * num_channels;
1426   }
1427 
1428   int16_t samples_removed;
1429   Accelerate::ReturnCodes return_code = accelerate_->Process(
1430       decoded_buffer, decoded_length, algorithm_buffer_.get(),
1431       &samples_removed);
1432   stats_.AcceleratedSamples(samples_removed);
1433   switch (return_code) {
1434     case Accelerate::kSuccess:
1435       last_mode_ = kModeAccelerateSuccess;
1436       break;
1437     case Accelerate::kSuccessLowEnergy:
1438       last_mode_ = kModeAccelerateLowEnergy;
1439       break;
1440     case Accelerate::kNoStretch:
1441       last_mode_ = kModeAccelerateFail;
1442       break;
1443     case Accelerate::kError:
1444       // TODO(hlundin): Map to kModeError instead?
1445       last_mode_ = kModeAccelerateFail;
1446       return kAccelerateError;
1447   }
1448 
1449   if (borrowed_samples_per_channel > 0) {
1450     // Copy borrowed samples back to the |sync_buffer_|.
1451     size_t length = algorithm_buffer_->Size();
1452     if (length < borrowed_samples_per_channel) {
1453       // This destroys the beginning of the buffer, but will not cause any
1454       // problems.
1455       sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1456                                    sync_buffer_->Size() -
1457                                    borrowed_samples_per_channel);
1458       sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length);
1459       algorithm_buffer_->PopFront(length);
1460       assert(algorithm_buffer_->Empty());
1461     } else {
1462       sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1463                                    borrowed_samples_per_channel,
1464                                    sync_buffer_->Size() -
1465                                    borrowed_samples_per_channel);
1466       algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1467     }
1468   }
1469 
1470   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1471   if (speech_type == AudioDecoder::kComfortNoise) {
1472     last_mode_ = kModeCodecInternalCng;
1473   }
1474   if (!play_dtmf) {
1475     dtmf_tone_generator_->Reset();
1476   }
1477   expand_->Reset();
1478   return 0;
1479 }
1480 
DoPreemptiveExpand(int16_t * decoded_buffer,size_t decoded_length,AudioDecoder::SpeechType speech_type,bool play_dtmf)1481 int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer,
1482                                   size_t decoded_length,
1483                                   AudioDecoder::SpeechType speech_type,
1484                                   bool play_dtmf) {
1485   const size_t required_samples = 240 * fs_mult_;  // Must have 30 ms.
1486   size_t num_channels = algorithm_buffer_->Channels();
1487   int borrowed_samples_per_channel = 0;
1488   int old_borrowed_samples_per_channel = 0;
1489   size_t decoded_length_per_channel = decoded_length / num_channels;
1490   if (decoded_length_per_channel < required_samples) {
1491     // Must move data from the |sync_buffer_| in order to get 30 ms.
1492     borrowed_samples_per_channel = static_cast<int>(required_samples -
1493         decoded_length_per_channel);
1494     // Calculate how many of these were already played out.
1495     old_borrowed_samples_per_channel = static_cast<int>(
1496         borrowed_samples_per_channel - sync_buffer_->FutureLength());
1497     old_borrowed_samples_per_channel = std::max(
1498         0, old_borrowed_samples_per_channel);
1499     memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1500             decoded_buffer,
1501             sizeof(int16_t) * decoded_length);
1502     sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1503                                          decoded_buffer);
1504     decoded_length = required_samples * num_channels;
1505   }
1506 
1507   int16_t samples_added;
1508   PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process(
1509       decoded_buffer, static_cast<int>(decoded_length),
1510       old_borrowed_samples_per_channel,
1511       algorithm_buffer_.get(), &samples_added);
1512   stats_.PreemptiveExpandedSamples(samples_added);
1513   switch (return_code) {
1514     case PreemptiveExpand::kSuccess:
1515       last_mode_ = kModePreemptiveExpandSuccess;
1516       break;
1517     case PreemptiveExpand::kSuccessLowEnergy:
1518       last_mode_ = kModePreemptiveExpandLowEnergy;
1519       break;
1520     case PreemptiveExpand::kNoStretch:
1521       last_mode_ = kModePreemptiveExpandFail;
1522       break;
1523     case PreemptiveExpand::kError:
1524       // TODO(hlundin): Map to kModeError instead?
1525       last_mode_ = kModePreemptiveExpandFail;
1526       return kPreemptiveExpandError;
1527   }
1528 
1529   if (borrowed_samples_per_channel > 0) {
1530     // Copy borrowed samples back to the |sync_buffer_|.
1531     sync_buffer_->ReplaceAtIndex(
1532         *algorithm_buffer_, borrowed_samples_per_channel,
1533         sync_buffer_->Size() - borrowed_samples_per_channel);
1534     algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1535   }
1536 
1537   // If last packet was decoded as an inband CNG, set mode to CNG instead.
1538   if (speech_type == AudioDecoder::kComfortNoise) {
1539     last_mode_ = kModeCodecInternalCng;
1540   }
1541   if (!play_dtmf) {
1542     dtmf_tone_generator_->Reset();
1543   }
1544   expand_->Reset();
1545   return 0;
1546 }
1547 
DoRfc3389Cng(PacketList * packet_list,bool play_dtmf)1548 int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) {
1549   if (!packet_list->empty()) {
1550     // Must have exactly one SID frame at this point.
1551     assert(packet_list->size() == 1);
1552     Packet* packet = packet_list->front();
1553     packet_list->pop_front();
1554     if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {
1555 #ifdef LEGACY_BITEXACT
1556       // This can happen due to a bug in GetDecision. Change the payload type
1557       // to a CNG type, and move on. Note that this means that we are in fact
1558       // sending a non-CNG payload to the comfort noise decoder for decoding.
1559       // Clearly wrong, but will maintain bit-exactness with legacy.
1560       if (fs_hz_ == 8000) {
1561         packet->header.payloadType =
1562             decoder_database_->GetRtpPayloadType(kDecoderCNGnb);
1563       } else if (fs_hz_ == 16000) {
1564         packet->header.payloadType =
1565             decoder_database_->GetRtpPayloadType(kDecoderCNGwb);
1566       } else if (fs_hz_ == 32000) {
1567         packet->header.payloadType =
1568             decoder_database_->GetRtpPayloadType(kDecoderCNGswb32kHz);
1569       } else if (fs_hz_ == 48000) {
1570         packet->header.payloadType =
1571             decoder_database_->GetRtpPayloadType(kDecoderCNGswb48kHz);
1572       }
1573       assert(decoder_database_->IsComfortNoise(packet->header.payloadType));
1574 #else
1575       LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG.";
1576       return kOtherError;
1577 #endif
1578     }
1579     // UpdateParameters() deletes |packet|.
1580     if (comfort_noise_->UpdateParameters(packet) ==
1581         ComfortNoise::kInternalError) {
1582       LOG_FERR0(LS_WARNING, UpdateParameters);
1583       algorithm_buffer_->Zeros(output_size_samples_);
1584       return -comfort_noise_->internal_error_code();
1585     }
1586   }
1587   int cn_return = comfort_noise_->Generate(output_size_samples_,
1588                                            algorithm_buffer_.get());
1589   expand_->Reset();
1590   last_mode_ = kModeRfc3389Cng;
1591   if (!play_dtmf) {
1592     dtmf_tone_generator_->Reset();
1593   }
1594   if (cn_return == ComfortNoise::kInternalError) {
1595     LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1596     decoder_error_code_ = comfort_noise_->internal_error_code();
1597     return kComfortNoiseErrorCode;
1598   } else if (cn_return == ComfortNoise::kUnknownPayloadType) {
1599     LOG_FERR1(LS_WARNING, comfort_noise_->Generate, cn_return);
1600     return kUnknownRtpPayloadType;
1601   }
1602   return 0;
1603 }
1604 
DoCodecInternalCng()1605 void NetEqImpl::DoCodecInternalCng() {
1606   int length = 0;
1607   // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1608   int16_t decoded_buffer[kMaxFrameSize];
1609   AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1610   if (decoder) {
1611     const uint8_t* dummy_payload = NULL;
1612     AudioDecoder::SpeechType speech_type;
1613     length = decoder->Decode(
1614         dummy_payload, 0, fs_hz_, kMaxFrameSize * sizeof(int16_t),
1615         decoded_buffer, &speech_type);
1616   }
1617   assert(mute_factor_array_.get());
1618   normal_->Process(decoded_buffer, length, last_mode_, mute_factor_array_.get(),
1619                    algorithm_buffer_.get());
1620   last_mode_ = kModeCodecInternalCng;
1621   expand_->Reset();
1622 }
1623 
DoDtmf(const DtmfEvent & dtmf_event,bool * play_dtmf)1624 int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) {
1625   // This block of the code and the block further down, handling |dtmf_switch|
1626   // are commented out. Otherwise playing out-of-band DTMF would fail in VoE
1627   // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is
1628   // equivalent to |dtmf_switch| always be false.
1629   //
1630   // See http://webrtc-codereview.appspot.com/1195004/ for discussion
1631   // On this issue. This change might cause some glitches at the point of
1632   // switch from audio to DTMF. Issue 1545 is filed to track this.
1633   //
1634   //  bool dtmf_switch = false;
1635   //  if ((last_mode_ != kModeDtmf) && dtmf_tone_generator_->initialized()) {
1636   //    // Special case; see below.
1637   //    // We must catch this before calling Generate, since |initialized| is
1638   //    // modified in that call.
1639   //    dtmf_switch = true;
1640   //  }
1641 
1642   int dtmf_return_value = 0;
1643   if (!dtmf_tone_generator_->initialized()) {
1644     // Initialize if not already done.
1645     dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1646                                                    dtmf_event.volume);
1647   }
1648 
1649   if (dtmf_return_value == 0) {
1650     // Generate DTMF signal.
1651     dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_,
1652                                                        algorithm_buffer_.get());
1653   }
1654 
1655   if (dtmf_return_value < 0) {
1656     algorithm_buffer_->Zeros(output_size_samples_);
1657     return dtmf_return_value;
1658   }
1659 
1660   //  if (dtmf_switch) {
1661   //    // This is the special case where the previous operation was DTMF
1662   //    // overdub, but the current instruction is "regular" DTMF. We must make
1663   //    // sure that the DTMF does not have any discontinuities. The first DTMF
1664   //    // sample that we generate now must be played out immediately, therefore
1665   //    // it must be copied to the speech buffer.
1666   //    // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and
1667   //    // verify correct operation.
1668   //    assert(false);
1669   //    // Must generate enough data to replace all of the |sync_buffer_|
1670   //    // "future".
1671   //    int required_length = sync_buffer_->FutureLength();
1672   //    assert(dtmf_tone_generator_->initialized());
1673   //    dtmf_return_value = dtmf_tone_generator_->Generate(required_length,
1674   //                                                       algorithm_buffer_);
1675   //    assert((size_t) required_length == algorithm_buffer_->Size());
1676   //    if (dtmf_return_value < 0) {
1677   //      algorithm_buffer_->Zeros(output_size_samples_);
1678   //      return dtmf_return_value;
1679   //    }
1680   //
1681   //    // Overwrite the "future" part of the speech buffer with the new DTMF
1682   //    // data.
1683   //    // TODO(hlundin): It seems that this overwriting has gone lost.
1684   //    // Not adapted for multi-channel yet.
1685   //    assert(algorithm_buffer_->Channels() == 1);
1686   //    if (algorithm_buffer_->Channels() != 1) {
1687   //      LOG(LS_WARNING) << "DTMF not supported for more than one channel";
1688   //      return kStereoNotSupported;
1689   //    }
1690   //    // Shuffle the remaining data to the beginning of algorithm buffer.
1691   //    algorithm_buffer_->PopFront(sync_buffer_->FutureLength());
1692   //  }
1693 
1694   sync_buffer_->IncreaseEndTimestamp(output_size_samples_);
1695   expand_->Reset();
1696   last_mode_ = kModeDtmf;
1697 
1698   // Set to false because the DTMF is already in the algorithm buffer.
1699   *play_dtmf = false;
1700   return 0;
1701 }
1702 
DoAlternativePlc(bool increase_timestamp)1703 void NetEqImpl::DoAlternativePlc(bool increase_timestamp) {
1704   AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1705   int length;
1706   if (decoder && decoder->HasDecodePlc()) {
1707     // Use the decoder's packet-loss concealment.
1708     // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1709     int16_t decoded_buffer[kMaxFrameSize];
1710     length = decoder->DecodePlc(1, decoded_buffer);
1711     if (length > 0) {
1712       algorithm_buffer_->PushBackInterleaved(decoded_buffer, length);
1713     } else {
1714       length = 0;
1715     }
1716   } else {
1717     // Do simple zero-stuffing.
1718     length = output_size_samples_;
1719     algorithm_buffer_->Zeros(length);
1720     // By not advancing the timestamp, NetEq inserts samples.
1721     stats_.AddZeros(length);
1722   }
1723   if (increase_timestamp) {
1724     sync_buffer_->IncreaseEndTimestamp(length);
1725   }
1726   expand_->Reset();
1727 }
1728 
DtmfOverdub(const DtmfEvent & dtmf_event,size_t num_channels,int16_t * output) const1729 int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels,
1730                            int16_t* output) const {
1731   size_t out_index = 0;
1732   int overdub_length = output_size_samples_;  // Default value.
1733 
1734   if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) {
1735     // Special operation for transition from "DTMF only" to "DTMF overdub".
1736     out_index = std::min(
1737         sync_buffer_->dtmf_index() - sync_buffer_->next_index(),
1738         static_cast<size_t>(output_size_samples_));
1739     overdub_length = output_size_samples_ - static_cast<int>(out_index);
1740   }
1741 
1742   AudioMultiVector dtmf_output(num_channels);
1743   int dtmf_return_value = 0;
1744   if (!dtmf_tone_generator_->initialized()) {
1745     dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1746                                                    dtmf_event.volume);
1747   }
1748   if (dtmf_return_value == 0) {
1749     dtmf_return_value = dtmf_tone_generator_->Generate(overdub_length,
1750                                                        &dtmf_output);
1751     assert((size_t) overdub_length == dtmf_output.Size());
1752   }
1753   dtmf_output.ReadInterleaved(overdub_length, &output[out_index]);
1754   return dtmf_return_value < 0 ? dtmf_return_value : 0;
1755 }
1756 
ExtractPackets(int required_samples,PacketList * packet_list)1757 int NetEqImpl::ExtractPackets(int required_samples, PacketList* packet_list) {
1758   bool first_packet = true;
1759   uint8_t prev_payload_type = 0;
1760   uint32_t prev_timestamp = 0;
1761   uint16_t prev_sequence_number = 0;
1762   bool next_packet_available = false;
1763 
1764   const RTPHeader* header = packet_buffer_->NextRtpHeader();
1765   assert(header);
1766   if (!header) {
1767     return -1;
1768   }
1769   uint32_t first_timestamp = header->timestamp;
1770   int extracted_samples = 0;
1771 
1772   // Packet extraction loop.
1773   do {
1774     timestamp_ = header->timestamp;
1775     int discard_count = 0;
1776     Packet* packet = packet_buffer_->GetNextPacket(&discard_count);
1777     // |header| may be invalid after the |packet_buffer_| operation.
1778     header = NULL;
1779     if (!packet) {
1780       LOG_FERR1(LS_ERROR, GetNextPacket, discard_count) <<
1781           "Should always be able to extract a packet here";
1782       assert(false);  // Should always be able to extract a packet here.
1783       return -1;
1784     }
1785     stats_.PacketsDiscarded(discard_count);
1786     // Store waiting time in ms; packets->waiting_time is in "output blocks".
1787     stats_.StoreWaitingTime(packet->waiting_time * kOutputSizeMs);
1788     assert(packet->payload_length > 0);
1789     packet_list->push_back(packet);  // Store packet in list.
1790 
1791     if (first_packet) {
1792       first_packet = false;
1793       decoded_packet_sequence_number_ = prev_sequence_number =
1794           packet->header.sequenceNumber;
1795       decoded_packet_timestamp_ = prev_timestamp = packet->header.timestamp;
1796       prev_payload_type = packet->header.payloadType;
1797     }
1798 
1799     // Store number of extracted samples.
1800     int packet_duration = 0;
1801     AudioDecoder* decoder = decoder_database_->GetDecoder(
1802         packet->header.payloadType);
1803     if (decoder) {
1804       if (packet->sync_packet) {
1805         packet_duration = decoder_frame_length_;
1806       } else {
1807         if (packet->primary) {
1808           packet_duration = decoder->PacketDuration(packet->payload,
1809                                                     packet->payload_length);
1810         } else {
1811           packet_duration = decoder->
1812               PacketDurationRedundant(packet->payload, packet->payload_length);
1813           stats_.SecondaryDecodedSamples(packet_duration);
1814         }
1815       }
1816     } else {
1817       LOG_FERR1(LS_WARNING, GetDecoder,
1818                 static_cast<int>(packet->header.payloadType))
1819           << "Could not find a decoder for a packet about to be extracted.";
1820       assert(false);
1821     }
1822     if (packet_duration <= 0) {
1823       // Decoder did not return a packet duration. Assume that the packet
1824       // contains the same number of samples as the previous one.
1825       packet_duration = decoder_frame_length_;
1826     }
1827     extracted_samples = packet->header.timestamp - first_timestamp +
1828         packet_duration;
1829 
1830     // Check what packet is available next.
1831     header = packet_buffer_->NextRtpHeader();
1832     next_packet_available = false;
1833     if (header && prev_payload_type == header->payloadType) {
1834       int16_t seq_no_diff = header->sequenceNumber - prev_sequence_number;
1835       int32_t ts_diff = header->timestamp - prev_timestamp;
1836       if (seq_no_diff == 1 ||
1837           (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) {
1838         // The next sequence number is available, or the next part of a packet
1839         // that was split into pieces upon insertion.
1840         next_packet_available = true;
1841       }
1842       prev_sequence_number = header->sequenceNumber;
1843     }
1844   } while (extracted_samples < required_samples && next_packet_available);
1845 
1846   if (extracted_samples > 0) {
1847     // Delete old packets only when we are going to decode something. Otherwise,
1848     // we could end up in the situation where we never decode anything, since
1849     // all incoming packets are considered too old but the buffer will also
1850     // never be flooded and flushed.
1851     packet_buffer_->DiscardAllOldPackets(timestamp_);
1852   }
1853 
1854   return extracted_samples;
1855 }
1856 
UpdatePlcComponents(int fs_hz,size_t channels)1857 void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) {
1858   // Delete objects and create new ones.
1859   expand_.reset(expand_factory_->Create(background_noise_.get(),
1860                                         sync_buffer_.get(), &random_vector_,
1861                                         fs_hz, channels));
1862   merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get()));
1863 }
1864 
SetSampleRateAndChannels(int fs_hz,size_t channels)1865 void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) {
1866   LOG_API2(fs_hz, channels);
1867   // TODO(hlundin): Change to an enumerator and skip assert.
1868   assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz ==  32000 || fs_hz == 48000);
1869   assert(channels > 0);
1870 
1871   fs_hz_ = fs_hz;
1872   fs_mult_ = fs_hz / 8000;
1873   output_size_samples_ = kOutputSizeMs * 8 * fs_mult_;
1874   decoder_frame_length_ = 3 * output_size_samples_;  // Initialize to 30ms.
1875 
1876   last_mode_ = kModeNormal;
1877 
1878   // Create a new array of mute factors and set all to 1.
1879   mute_factor_array_.reset(new int16_t[channels]);
1880   for (size_t i = 0; i < channels; ++i) {
1881     mute_factor_array_[i] = 16384;  // 1.0 in Q14.
1882   }
1883 
1884   // Reset comfort noise decoder, if there is one active.
1885   AudioDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1886   if (cng_decoder) {
1887     cng_decoder->Init();
1888   }
1889 
1890   // Reinit post-decode VAD with new sample rate.
1891   assert(vad_.get());  // Cannot be NULL here.
1892   vad_->Init();
1893 
1894   // Delete algorithm buffer and create a new one.
1895   algorithm_buffer_.reset(new AudioMultiVector(channels));
1896 
1897   // Delete sync buffer and create a new one.
1898   sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_));
1899 
1900   // Delete BackgroundNoise object and create a new one.
1901   background_noise_.reset(new BackgroundNoise(channels));
1902   background_noise_->set_mode(background_noise_mode_);
1903 
1904   // Reset random vector.
1905   random_vector_.Reset();
1906 
1907   UpdatePlcComponents(fs_hz, channels);
1908 
1909   // Move index so that we create a small set of future samples (all 0).
1910   sync_buffer_->set_next_index(sync_buffer_->next_index() -
1911       expand_->overlap_length());
1912 
1913   normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_,
1914                            expand_.get()));
1915   accelerate_.reset(
1916       accelerate_factory_->Create(fs_hz, channels, *background_noise_));
1917   preemptive_expand_.reset(preemptive_expand_factory_->Create(
1918       fs_hz, channels,
1919       *background_noise_,
1920       static_cast<int>(expand_->overlap_length())));
1921 
1922   // Delete ComfortNoise object and create a new one.
1923   comfort_noise_.reset(new ComfortNoise(fs_hz, decoder_database_.get(),
1924                                         sync_buffer_.get()));
1925 
1926   // Verify that |decoded_buffer_| is long enough.
1927   if (decoded_buffer_length_ < kMaxFrameSize * channels) {
1928     // Reallocate to larger size.
1929     decoded_buffer_length_ = kMaxFrameSize * channels;
1930     decoded_buffer_.reset(new int16_t[decoded_buffer_length_]);
1931   }
1932 
1933   // Create DecisionLogic if it is not created yet, then communicate new sample
1934   // rate and output size to DecisionLogic object.
1935   if (!decision_logic_.get()) {
1936     CreateDecisionLogic();
1937   }
1938   decision_logic_->SetSampleRate(fs_hz_, output_size_samples_);
1939 }
1940 
LastOutputType()1941 NetEqOutputType NetEqImpl::LastOutputType() {
1942   assert(vad_.get());
1943   assert(expand_.get());
1944   if (last_mode_ == kModeCodecInternalCng || last_mode_ == kModeRfc3389Cng) {
1945     return kOutputCNG;
1946   } else if (last_mode_ == kModeExpand && expand_->MuteFactor(0) == 0) {
1947     // Expand mode has faded down to background noise only (very long expand).
1948     return kOutputPLCtoCNG;
1949   } else if (last_mode_ == kModeExpand) {
1950     return kOutputPLC;
1951   } else if (vad_->running() && !vad_->active_speech()) {
1952     return kOutputVADPassive;
1953   } else {
1954     return kOutputNormal;
1955   }
1956 }
1957 
CreateDecisionLogic()1958 void NetEqImpl::CreateDecisionLogic() {
1959   decision_logic_.reset(DecisionLogic::Create(fs_hz_, output_size_samples_,
1960                                               playout_mode_,
1961                                               decoder_database_.get(),
1962                                               *packet_buffer_.get(),
1963                                               delay_manager_.get(),
1964                                               buffer_level_filter_.get()));
1965 }
1966 }  // namespace webrtc
1967