1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 12 #define WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 13 14 #include "webrtc/base/constructormagic.h" 15 #include "webrtc/base/scoped_ptr.h" 16 #include "webrtc/base/thread_annotations.h" 17 #include "webrtc/modules/audio_coding/neteq/audio_multi_vector.h" 18 #include "webrtc/modules/audio_coding/neteq/defines.h" 19 #include "webrtc/modules/audio_coding/neteq/include/neteq.h" 20 #include "webrtc/modules/audio_coding/neteq/packet.h" // Declare PacketList. 21 #include "webrtc/modules/audio_coding/neteq/random_vector.h" 22 #include "webrtc/modules/audio_coding/neteq/rtcp.h" 23 #include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" 24 #include "webrtc/typedefs.h" 25 26 namespace webrtc { 27 28 // Forward declarations. 29 class Accelerate; 30 class BackgroundNoise; 31 class BufferLevelFilter; 32 class ComfortNoise; 33 class CriticalSectionWrapper; 34 class DecisionLogic; 35 class DecoderDatabase; 36 class DelayManager; 37 class DelayPeakDetector; 38 class DtmfBuffer; 39 class DtmfToneGenerator; 40 class Expand; 41 class Merge; 42 class Nack; 43 class Normal; 44 class PacketBuffer; 45 class PayloadSplitter; 46 class PostDecodeVad; 47 class PreemptiveExpand; 48 class RandomVector; 49 class SyncBuffer; 50 class TimestampScaler; 51 struct AccelerateFactory; 52 struct DtmfEvent; 53 struct ExpandFactory; 54 struct PreemptiveExpandFactory; 55 56 class NetEqImpl : public webrtc::NetEq { 57 public: 58 // Creates a new NetEqImpl object. The object will assume ownership of all 59 // injected dependencies, and will delete them when done. 60 NetEqImpl(const NetEq::Config& config, 61 BufferLevelFilter* buffer_level_filter, 62 DecoderDatabase* decoder_database, 63 DelayManager* delay_manager, 64 DelayPeakDetector* delay_peak_detector, 65 DtmfBuffer* dtmf_buffer, 66 DtmfToneGenerator* dtmf_tone_generator, 67 PacketBuffer* packet_buffer, 68 PayloadSplitter* payload_splitter, 69 TimestampScaler* timestamp_scaler, 70 AccelerateFactory* accelerate_factory, 71 ExpandFactory* expand_factory, 72 PreemptiveExpandFactory* preemptive_expand_factory, 73 bool create_components = true); 74 75 ~NetEqImpl() override; 76 77 // Inserts a new packet into NetEq. The |receive_timestamp| is an indication 78 // of the time when the packet was received, and should be measured with 79 // the same tick rate as the RTP timestamp of the current payload. 80 // Returns 0 on success, -1 on failure. 81 int InsertPacket(const WebRtcRTPHeader& rtp_header, 82 const uint8_t* payload, 83 size_t length_bytes, 84 uint32_t receive_timestamp) override; 85 86 // Inserts a sync-packet into packet queue. Sync-packets are decoded to 87 // silence and are intended to keep AV-sync intact in an event of long packet 88 // losses when Video NACK is enabled but Audio NACK is not. Clients of NetEq 89 // might insert sync-packet when they observe that buffer level of NetEq is 90 // decreasing below a certain threshold, defined by the application. 91 // Sync-packets should have the same payload type as the last audio payload 92 // type, i.e. they cannot have DTMF or CNG payload type, nor a codec change 93 // can be implied by inserting a sync-packet. 94 // Returns kOk on success, kFail on failure. 95 int InsertSyncPacket(const WebRtcRTPHeader& rtp_header, 96 uint32_t receive_timestamp) override; 97 98 // Instructs NetEq to deliver 10 ms of audio data. The data is written to 99 // |output_audio|, which can hold (at least) |max_length| elements. 100 // The number of channels that were written to the output is provided in 101 // the output variable |num_channels|, and each channel contains 102 // |samples_per_channel| elements. If more than one channel is written, 103 // the samples are interleaved. 104 // The speech type is written to |type|, if |type| is not NULL. 105 // Returns kOK on success, or kFail in case of an error. 106 int GetAudio(size_t max_length, 107 int16_t* output_audio, 108 size_t* samples_per_channel, 109 int* num_channels, 110 NetEqOutputType* type) override; 111 112 // Associates |rtp_payload_type| with |codec| and stores the information in 113 // the codec database. Returns kOK on success, kFail on failure. 114 int RegisterPayloadType(NetEqDecoder codec, 115 uint8_t rtp_payload_type) override; 116 117 // Provides an externally created decoder object |decoder| to insert in the 118 // decoder database. The decoder implements a decoder of type |codec| and 119 // associates it with |rtp_payload_type|. The decoder will produce samples 120 // at the rate |sample_rate_hz|. Returns kOK on success, kFail on failure. 121 int RegisterExternalDecoder(AudioDecoder* decoder, 122 NetEqDecoder codec, 123 uint8_t rtp_payload_type, 124 int sample_rate_hz) override; 125 126 // Removes |rtp_payload_type| from the codec database. Returns 0 on success, 127 // -1 on failure. 128 int RemovePayloadType(uint8_t rtp_payload_type) override; 129 130 bool SetMinimumDelay(int delay_ms) override; 131 132 bool SetMaximumDelay(int delay_ms) override; 133 134 int LeastRequiredDelayMs() const override; 135 136 int SetTargetDelay() override; 137 138 int TargetDelay() override; 139 140 int CurrentDelayMs() const override; 141 142 // Sets the playout mode to |mode|. 143 // Deprecated. 144 // TODO(henrik.lundin) Delete. 145 void SetPlayoutMode(NetEqPlayoutMode mode) override; 146 147 // Returns the current playout mode. 148 // Deprecated. 149 // TODO(henrik.lundin) Delete. 150 NetEqPlayoutMode PlayoutMode() const override; 151 152 // Writes the current network statistics to |stats|. The statistics are reset 153 // after the call. 154 int NetworkStatistics(NetEqNetworkStatistics* stats) override; 155 156 // Writes the current RTCP statistics to |stats|. The statistics are reset 157 // and a new report period is started with the call. 158 void GetRtcpStatistics(RtcpStatistics* stats) override; 159 160 // Same as RtcpStatistics(), but does not reset anything. 161 void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override; 162 163 // Enables post-decode VAD. When enabled, GetAudio() will return 164 // kOutputVADPassive when the signal contains no speech. 165 void EnableVad() override; 166 167 // Disables post-decode VAD. 168 void DisableVad() override; 169 170 bool GetPlayoutTimestamp(uint32_t* timestamp) override; 171 172 int SetTargetNumberOfChannels() override; 173 174 int SetTargetSampleRate() override; 175 176 // Returns the error code for the last occurred error. If no error has 177 // occurred, 0 is returned. 178 int LastError() const override; 179 180 // Returns the error code last returned by a decoder (audio or comfort noise). 181 // When LastError() returns kDecoderErrorCode or kComfortNoiseErrorCode, check 182 // this method to get the decoder's error code. 183 int LastDecoderError() override; 184 185 // Flushes both the packet buffer and the sync buffer. 186 void FlushBuffers() override; 187 188 void PacketBufferStatistics(int* current_num_packets, 189 int* max_num_packets) const override; 190 191 void EnableNack(size_t max_nack_list_size) override; 192 193 void DisableNack() override; 194 195 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override; 196 197 // This accessor method is only intended for testing purposes. 198 const SyncBuffer* sync_buffer_for_test() const; 199 200 protected: 201 static const int kOutputSizeMs = 10; 202 static const size_t kMaxFrameSize = 2880; // 60 ms @ 48 kHz. 203 // TODO(hlundin): Provide a better value for kSyncBufferSize. 204 static const size_t kSyncBufferSize = 2 * kMaxFrameSize; 205 206 // Inserts a new packet into NetEq. This is used by the InsertPacket method 207 // above. Returns 0 on success, otherwise an error code. 208 // TODO(hlundin): Merge this with InsertPacket above? 209 int InsertPacketInternal(const WebRtcRTPHeader& rtp_header, 210 const uint8_t* payload, 211 size_t length_bytes, 212 uint32_t receive_timestamp, 213 bool is_sync_packet) 214 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 215 216 // Delivers 10 ms of audio data. The data is written to |output|, which can 217 // hold (at least) |max_length| elements. The number of channels that were 218 // written to the output is provided in the output variable |num_channels|, 219 // and each channel contains |samples_per_channel| elements. If more than one 220 // channel is written, the samples are interleaved. 221 // Returns 0 on success, otherwise an error code. 222 int GetAudioInternal(size_t max_length, 223 int16_t* output, 224 size_t* samples_per_channel, 225 int* num_channels) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 226 227 // Provides a decision to the GetAudioInternal method. The decision what to 228 // do is written to |operation|. Packets to decode are written to 229 // |packet_list|, and a DTMF event to play is written to |dtmf_event|. When 230 // DTMF should be played, |play_dtmf| is set to true by the method. 231 // Returns 0 on success, otherwise an error code. 232 int GetDecision(Operations* operation, 233 PacketList* packet_list, 234 DtmfEvent* dtmf_event, 235 bool* play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 236 237 // Decodes the speech packets in |packet_list|, and writes the results to 238 // |decoded_buffer|, which is allocated to hold |decoded_buffer_length| 239 // elements. The length of the decoded data is written to |decoded_length|. 240 // The speech type -- speech or (codec-internal) comfort noise -- is written 241 // to |speech_type|. If |packet_list| contains any SID frames for RFC 3389 242 // comfort noise, those are not decoded. 243 int Decode(PacketList* packet_list, 244 Operations* operation, 245 int* decoded_length, 246 AudioDecoder::SpeechType* speech_type) 247 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 248 249 // Sub-method to Decode(). Performs codec internal CNG. 250 int DecodeCng(AudioDecoder* decoder, int* decoded_length, 251 AudioDecoder::SpeechType* speech_type) 252 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 253 254 // Sub-method to Decode(). Performs the actual decoding. 255 int DecodeLoop(PacketList* packet_list, 256 const Operations& operation, 257 AudioDecoder* decoder, 258 int* decoded_length, 259 AudioDecoder::SpeechType* speech_type) 260 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 261 262 // Sub-method which calls the Normal class to perform the normal operation. 263 void DoNormal(const int16_t* decoded_buffer, 264 size_t decoded_length, 265 AudioDecoder::SpeechType speech_type, 266 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 267 268 // Sub-method which calls the Merge class to perform the merge operation. 269 void DoMerge(int16_t* decoded_buffer, 270 size_t decoded_length, 271 AudioDecoder::SpeechType speech_type, 272 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 273 274 // Sub-method which calls the Expand class to perform the expand operation. 275 int DoExpand(bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 276 277 // Sub-method which calls the Accelerate class to perform the accelerate 278 // operation. 279 int DoAccelerate(int16_t* decoded_buffer, 280 size_t decoded_length, 281 AudioDecoder::SpeechType speech_type, 282 bool play_dtmf, 283 bool fast_accelerate) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 284 285 // Sub-method which calls the PreemptiveExpand class to perform the 286 // preemtive expand operation. 287 int DoPreemptiveExpand(int16_t* decoded_buffer, 288 size_t decoded_length, 289 AudioDecoder::SpeechType speech_type, 290 bool play_dtmf) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 291 292 // Sub-method which calls the ComfortNoise class to generate RFC 3389 comfort 293 // noise. |packet_list| can either contain one SID frame to update the 294 // noise parameters, or no payload at all, in which case the previously 295 // received parameters are used. 296 int DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) 297 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 298 299 // Calls the audio decoder to generate codec-internal comfort noise when 300 // no packet was received. 301 void DoCodecInternalCng(const int16_t* decoded_buffer, size_t decoded_length) 302 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 303 304 // Calls the DtmfToneGenerator class to generate DTMF tones. 305 int DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) 306 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 307 308 // Produces packet-loss concealment using alternative methods. If the codec 309 // has an internal PLC, it is called to generate samples. Otherwise, the 310 // method performs zero-stuffing. 311 void DoAlternativePlc(bool increase_timestamp) 312 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 313 314 // Overdub DTMF on top of |output|. 315 int DtmfOverdub(const DtmfEvent& dtmf_event, 316 size_t num_channels, 317 int16_t* output) const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 318 319 // Extracts packets from |packet_buffer_| to produce at least 320 // |required_samples| samples. The packets are inserted into |packet_list|. 321 // Returns the number of samples that the packets in the list will produce, or 322 // -1 in case of an error. 323 int ExtractPackets(size_t required_samples, PacketList* packet_list) 324 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 325 326 // Resets various variables and objects to new values based on the sample rate 327 // |fs_hz| and |channels| number audio channels. 328 void SetSampleRateAndChannels(int fs_hz, size_t channels) 329 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 330 331 // Returns the output type for the audio produced by the latest call to 332 // GetAudio(). 333 NetEqOutputType LastOutputType() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 334 335 // Updates Expand and Merge. 336 virtual void UpdatePlcComponents(int fs_hz, size_t channels) 337 EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 338 339 // Creates DecisionLogic object with the mode given by |playout_mode_|. 340 virtual void CreateDecisionLogic() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); 341 342 const rtc::scoped_ptr<CriticalSectionWrapper> crit_sect_; 343 const rtc::scoped_ptr<BufferLevelFilter> buffer_level_filter_ 344 GUARDED_BY(crit_sect_); 345 const rtc::scoped_ptr<DecoderDatabase> decoder_database_ 346 GUARDED_BY(crit_sect_); 347 const rtc::scoped_ptr<DelayManager> delay_manager_ GUARDED_BY(crit_sect_); 348 const rtc::scoped_ptr<DelayPeakDetector> delay_peak_detector_ 349 GUARDED_BY(crit_sect_); 350 const rtc::scoped_ptr<DtmfBuffer> dtmf_buffer_ GUARDED_BY(crit_sect_); 351 const rtc::scoped_ptr<DtmfToneGenerator> dtmf_tone_generator_ 352 GUARDED_BY(crit_sect_); 353 const rtc::scoped_ptr<PacketBuffer> packet_buffer_ GUARDED_BY(crit_sect_); 354 const rtc::scoped_ptr<PayloadSplitter> payload_splitter_ 355 GUARDED_BY(crit_sect_); 356 const rtc::scoped_ptr<TimestampScaler> timestamp_scaler_ 357 GUARDED_BY(crit_sect_); 358 const rtc::scoped_ptr<PostDecodeVad> vad_ GUARDED_BY(crit_sect_); 359 const rtc::scoped_ptr<ExpandFactory> expand_factory_ GUARDED_BY(crit_sect_); 360 const rtc::scoped_ptr<AccelerateFactory> accelerate_factory_ 361 GUARDED_BY(crit_sect_); 362 const rtc::scoped_ptr<PreemptiveExpandFactory> preemptive_expand_factory_ 363 GUARDED_BY(crit_sect_); 364 365 rtc::scoped_ptr<BackgroundNoise> background_noise_ GUARDED_BY(crit_sect_); 366 rtc::scoped_ptr<DecisionLogic> decision_logic_ GUARDED_BY(crit_sect_); 367 rtc::scoped_ptr<AudioMultiVector> algorithm_buffer_ GUARDED_BY(crit_sect_); 368 rtc::scoped_ptr<SyncBuffer> sync_buffer_ GUARDED_BY(crit_sect_); 369 rtc::scoped_ptr<Expand> expand_ GUARDED_BY(crit_sect_); 370 rtc::scoped_ptr<Normal> normal_ GUARDED_BY(crit_sect_); 371 rtc::scoped_ptr<Merge> merge_ GUARDED_BY(crit_sect_); 372 rtc::scoped_ptr<Accelerate> accelerate_ GUARDED_BY(crit_sect_); 373 rtc::scoped_ptr<PreemptiveExpand> preemptive_expand_ GUARDED_BY(crit_sect_); 374 RandomVector random_vector_ GUARDED_BY(crit_sect_); 375 rtc::scoped_ptr<ComfortNoise> comfort_noise_ GUARDED_BY(crit_sect_); 376 Rtcp rtcp_ GUARDED_BY(crit_sect_); 377 StatisticsCalculator stats_ GUARDED_BY(crit_sect_); 378 int fs_hz_ GUARDED_BY(crit_sect_); 379 int fs_mult_ GUARDED_BY(crit_sect_); 380 size_t output_size_samples_ GUARDED_BY(crit_sect_); 381 size_t decoder_frame_length_ GUARDED_BY(crit_sect_); 382 Modes last_mode_ GUARDED_BY(crit_sect_); 383 rtc::scoped_ptr<int16_t[]> mute_factor_array_ GUARDED_BY(crit_sect_); 384 size_t decoded_buffer_length_ GUARDED_BY(crit_sect_); 385 rtc::scoped_ptr<int16_t[]> decoded_buffer_ GUARDED_BY(crit_sect_); 386 uint32_t playout_timestamp_ GUARDED_BY(crit_sect_); 387 bool new_codec_ GUARDED_BY(crit_sect_); 388 uint32_t timestamp_ GUARDED_BY(crit_sect_); 389 bool reset_decoder_ GUARDED_BY(crit_sect_); 390 uint8_t current_rtp_payload_type_ GUARDED_BY(crit_sect_); 391 uint8_t current_cng_rtp_payload_type_ GUARDED_BY(crit_sect_); 392 uint32_t ssrc_ GUARDED_BY(crit_sect_); 393 bool first_packet_ GUARDED_BY(crit_sect_); 394 int error_code_ GUARDED_BY(crit_sect_); // Store last error code. 395 int decoder_error_code_ GUARDED_BY(crit_sect_); 396 const BackgroundNoiseMode background_noise_mode_ GUARDED_BY(crit_sect_); 397 NetEqPlayoutMode playout_mode_ GUARDED_BY(crit_sect_); 398 bool enable_fast_accelerate_ GUARDED_BY(crit_sect_); 399 rtc::scoped_ptr<Nack> nack_ GUARDED_BY(crit_sect_); 400 bool nack_enabled_ GUARDED_BY(crit_sect_); 401 402 private: 403 RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl); 404 }; 405 406 } // namespace webrtc 407 #endif // WEBRTC_MODULES_AUDIO_CODING_NETEQ_NETEQ_IMPL_H_ 408