1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 12 #define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 13 14 #include <list> 15 #include <memory> 16 #include <vector> 17 18 #include "modules/audio_processing/audio_buffer.h" 19 #include "modules/audio_processing/include/aec_dump.h" 20 #include "modules/audio_processing/include/audio_processing.h" 21 #include "modules/audio_processing/render_queue_item_verifier.h" 22 #include "modules/audio_processing/rms_level.h" 23 #include "rtc_base/criticalsection.h" 24 #include "rtc_base/function_view.h" 25 #include "rtc_base/gtest_prod_util.h" 26 #include "rtc_base/ignore_wundef.h" 27 #include "rtc_base/protobuf_utils.h" 28 #include "rtc_base/swap_queue.h" 29 #include "rtc_base/thread_annotations.h" 30 #include "system_wrappers/include/file_wrapper.h" 31 32 namespace webrtc { 33 34 class AudioConverter; 35 class NonlinearBeamformer; 36 37 class AudioProcessingImpl : public AudioProcessing { 38 public: 39 // Methods forcing APM to run in a single-threaded manner. 40 // Acquires both the render and capture locks. 41 explicit AudioProcessingImpl(const webrtc::Config& config); 42 // AudioProcessingImpl takes ownership of capture post processor and 43 // beamformer. 44 AudioProcessingImpl(const webrtc::Config& config, 45 std::unique_ptr<PostProcessing> capture_post_processor, 46 std::unique_ptr<EchoControlFactory> echo_control_factory, 47 NonlinearBeamformer* beamformer); 48 ~AudioProcessingImpl() override; 49 int Initialize() override; 50 int Initialize(int capture_input_sample_rate_hz, 51 int capture_output_sample_rate_hz, 52 int render_sample_rate_hz, 53 ChannelLayout capture_input_layout, 54 ChannelLayout capture_output_layout, 55 ChannelLayout render_input_layout) override; 56 int Initialize(const ProcessingConfig& processing_config) override; 57 void ApplyConfig(const AudioProcessing::Config& config) override; 58 void SetExtraOptions(const webrtc::Config& config) override; 59 void UpdateHistogramsOnCallEnd() override; 60 void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override; 61 void DetachAecDump() override; 62 63 // Capture-side exclusive methods possibly running APM in a 64 // multi-threaded manner. Acquire the capture lock. 65 int ProcessStream(AudioFrame* frame) override; 66 int ProcessStream(const float* const* src, 67 size_t samples_per_channel, 68 int input_sample_rate_hz, 69 ChannelLayout input_layout, 70 int output_sample_rate_hz, 71 ChannelLayout output_layout, 72 float* const* dest) override; 73 int ProcessStream(const float* const* src, 74 const StreamConfig& input_config, 75 const StreamConfig& output_config, 76 float* const* dest) override; 77 void set_output_will_be_muted(bool muted) override; 78 int set_stream_delay_ms(int delay) override; 79 void set_delay_offset_ms(int offset) override; 80 int delay_offset_ms() const override; 81 void set_stream_key_pressed(bool key_pressed) override; 82 83 // Render-side exclusive methods possibly running APM in a 84 // multi-threaded manner. Acquire the render lock. 85 int ProcessReverseStream(AudioFrame* frame) override; 86 int AnalyzeReverseStream(const float* const* data, 87 size_t samples_per_channel, 88 int sample_rate_hz, 89 ChannelLayout layout) override; 90 int ProcessReverseStream(const float* const* src, 91 const StreamConfig& input_config, 92 const StreamConfig& output_config, 93 float* const* dest) override; 94 95 // Methods only accessed from APM submodules or 96 // from AudioProcessing tests in a single-threaded manner. 97 // Hence there is no need for locks in these. 98 int proc_sample_rate_hz() const override; 99 int proc_split_sample_rate_hz() const override; 100 size_t num_input_channels() const override; 101 size_t num_proc_channels() const override; 102 size_t num_output_channels() const override; 103 size_t num_reverse_channels() const override; 104 int stream_delay_ms() const override; 105 bool was_stream_delay_set() const override 106 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 107 108 AudioProcessingStatistics GetStatistics() const override; 109 AudioProcessingStats GetStatistics(bool has_remote_tracks) const override; 110 111 // Methods returning pointers to APM submodules. 112 // No locks are aquired in those, as those locks 113 // would offer no protection (the submodules are 114 // created only once in a single-treaded manner 115 // during APM creation). 116 EchoCancellation* echo_cancellation() const override; 117 EchoControlMobile* echo_control_mobile() const override; 118 GainControl* gain_control() const override; 119 // TODO(peah): Deprecate this API call. 120 HighPassFilter* high_pass_filter() const override; 121 LevelEstimator* level_estimator() const override; 122 NoiseSuppression* noise_suppression() const override; 123 VoiceDetection* voice_detection() const override; 124 125 // TODO(peah): Remove MutateConfig once the new API allows that. 126 void MutateConfig(rtc::FunctionView<void(AudioProcessing::Config*)> mutator); 127 AudioProcessing::Config GetConfig() const override; 128 129 protected: 130 // Overridden in a mock. 131 virtual int InitializeLocked() 132 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 133 134 private: 135 // TODO(peah): These friend classes should be removed as soon as the new 136 // parameter setting scheme allows. 137 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior); 138 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior); 139 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior); 140 struct ApmPublicSubmodules; 141 struct ApmPrivateSubmodules; 142 143 // Submodule interface implementations. 144 std::unique_ptr<HighPassFilter> high_pass_filter_impl_; 145 146 // EchoControl factory. 147 std::unique_ptr<EchoControlFactory> echo_control_factory_; 148 149 class ApmSubmoduleStates { 150 public: 151 explicit ApmSubmoduleStates(bool capture_post_processor_enabled); 152 // Updates the submodule state and returns true if it has changed. 153 bool Update(bool low_cut_filter_enabled, 154 bool echo_canceller_enabled, 155 bool mobile_echo_controller_enabled, 156 bool residual_echo_detector_enabled, 157 bool noise_suppressor_enabled, 158 bool intelligibility_enhancer_enabled, 159 bool beamformer_enabled, 160 bool adaptive_gain_controller_enabled, 161 bool gain_controller2_enabled, 162 bool level_controller_enabled, 163 bool echo_controller_enabled, 164 bool voice_activity_detector_enabled, 165 bool level_estimator_enabled, 166 bool transient_suppressor_enabled); 167 bool CaptureMultiBandSubModulesActive() const; 168 bool CaptureMultiBandProcessingActive() const; 169 bool CaptureFullBandProcessingActive() const; 170 bool RenderMultiBandSubModulesActive() const; 171 bool RenderMultiBandProcessingActive() const; 172 173 private: 174 const bool capture_post_processor_enabled_ = false; 175 bool low_cut_filter_enabled_ = false; 176 bool echo_canceller_enabled_ = false; 177 bool mobile_echo_controller_enabled_ = false; 178 bool residual_echo_detector_enabled_ = false; 179 bool noise_suppressor_enabled_ = false; 180 bool intelligibility_enhancer_enabled_ = false; 181 bool beamformer_enabled_ = false; 182 bool adaptive_gain_controller_enabled_ = false; 183 bool gain_controller2_enabled_ = false; 184 bool level_controller_enabled_ = false; 185 bool echo_controller_enabled_ = false; 186 bool level_estimator_enabled_ = false; 187 bool voice_activity_detector_enabled_ = false; 188 bool transient_suppressor_enabled_ = false; 189 bool first_update_ = true; 190 }; 191 192 // Method for modifying the formats struct that are called from both 193 // the render and capture threads. The check for whether modifications 194 // are needed is done while holding the render lock only, thereby avoiding 195 // that the capture thread blocks the render thread. 196 // The struct is modified in a single-threaded manner by holding both the 197 // render and capture locks. 198 int MaybeInitialize(const ProcessingConfig& config, bool force_initialization) 199 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 200 201 int MaybeInitializeRender(const ProcessingConfig& processing_config) 202 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 203 204 int MaybeInitializeCapture(const ProcessingConfig& processing_config, 205 bool force_initialization) 206 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 207 208 // Method for updating the state keeping track of the active submodules. 209 // Returns a bool indicating whether the state has changed. 210 bool UpdateActiveSubmoduleStates() 211 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 212 213 // Methods requiring APM running in a single-threaded manner. 214 // Are called with both the render and capture locks already 215 // acquired. 216 void InitializeTransient() 217 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 218 void InitializeBeamformer() 219 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 220 void InitializeIntelligibility() 221 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 222 int InitializeLocked(const ProcessingConfig& config) 223 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 224 void InitializeLevelController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 225 void InitializeResidualEchoDetector() 226 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 227 void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 228 void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 229 void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 230 void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 231 232 void EmptyQueuedRenderAudio(); 233 void AllocateRenderQueue() 234 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); 235 void QueueBandedRenderAudio(AudioBuffer* audio) 236 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 237 void QueueNonbandedRenderAudio(AudioBuffer* audio) 238 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 239 240 // Capture-side exclusive methods possibly running APM in a multi-threaded 241 // manner that are called with the render lock already acquired. 242 int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 243 void MaybeUpdateHistograms() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 244 245 // Render-side exclusive methods possibly running APM in a multi-threaded 246 // manner that are called with the render lock already acquired. 247 // TODO(ekm): Remove once all clients updated to new interface. 248 int AnalyzeReverseStreamLocked(const float* const* src, 249 const StreamConfig& input_config, 250 const StreamConfig& output_config) 251 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 252 int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); 253 254 // Collects configuration settings from public and private 255 // submodules to be saved as an audioproc::Config message on the 256 // AecDump if it is attached. If not |forced|, only writes the current 257 // config if it is different from the last saved one; if |forced|, 258 // writes the config regardless of the last saved. 259 void WriteAecDumpConfigMessage(bool forced) 260 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 261 262 // Notifies attached AecDump of current configuration and capture data. 263 void RecordUnprocessedCaptureStream(const float* const* capture_stream) 264 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 265 266 void RecordUnprocessedCaptureStream(const AudioFrame& capture_frame) 267 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 268 269 // Notifies attached AecDump of current configuration and 270 // processed capture data and issues a capture stream recording 271 // request. 272 void RecordProcessedCaptureStream( 273 const float* const* processed_capture_stream) 274 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 275 276 void RecordProcessedCaptureStream(const AudioFrame& processed_capture_frame) 277 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 278 279 // Notifies attached AecDump about current state (delay, drift, etc). 280 void RecordAudioProcessingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); 281 282 // AecDump instance used for optionally logging APM config, input 283 // and output to file in the AEC-dump format defined in debug.proto. 284 std::unique_ptr<AecDump> aec_dump_; 285 286 // Hold the last config written with AecDump for avoiding writing 287 // the same config twice. 288 InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(crit_capture_); 289 290 // Critical sections. 291 rtc::CriticalSection crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_); 292 rtc::CriticalSection crit_capture_; 293 294 // Struct containing the Config specifying the behavior of APM. 295 AudioProcessing::Config config_; 296 297 // Class containing information about what submodules are active. 298 ApmSubmoduleStates submodule_states_; 299 300 // Structs containing the pointers to the submodules. 301 std::unique_ptr<ApmPublicSubmodules> public_submodules_; 302 std::unique_ptr<ApmPrivateSubmodules> private_submodules_; 303 304 // State that is written to while holding both the render and capture locks 305 // but can be read without any lock being held. 306 // As this is only accessed internally of APM, and all internal methods in APM 307 // either are holding the render or capture locks, this construct is safe as 308 // it is not possible to read the variables while writing them. 309 struct ApmFormatState { ApmFormatStateApmFormatState310 ApmFormatState() 311 : // Format of processing streams at input/output call sites. 312 api_format({{{kSampleRate16kHz, 1, false}, 313 {kSampleRate16kHz, 1, false}, 314 {kSampleRate16kHz, 1, false}, 315 {kSampleRate16kHz, 1, false}}}), 316 render_processing_format(kSampleRate16kHz, 1) {} 317 ProcessingConfig api_format; 318 StreamConfig render_processing_format; 319 } formats_; 320 321 // APM constants. 322 const struct ApmConstants { ApmConstantsApmConstants323 ApmConstants(int agc_startup_min_volume, 324 int agc_clipped_level_min, 325 bool use_experimental_agc) 326 : // Format of processing streams at input/output call sites. 327 agc_startup_min_volume(agc_startup_min_volume), 328 agc_clipped_level_min(agc_clipped_level_min), 329 use_experimental_agc(use_experimental_agc) {} 330 int agc_startup_min_volume; 331 int agc_clipped_level_min; 332 bool use_experimental_agc; 333 } constants_; 334 335 struct ApmCaptureState { 336 ApmCaptureState(bool transient_suppressor_enabled, 337 const std::vector<Point>& array_geometry, 338 SphericalPointf target_direction); 339 ~ApmCaptureState(); 340 int aec_system_delay_jumps; 341 int delay_offset_ms; 342 bool was_stream_delay_set; 343 int last_stream_delay_ms; 344 int last_aec_system_delay_ms; 345 int stream_delay_jumps; 346 bool output_will_be_muted; 347 bool key_pressed; 348 bool transient_suppressor_enabled; 349 std::vector<Point> array_geometry; 350 SphericalPointf target_direction; 351 std::unique_ptr<AudioBuffer> capture_audio; 352 // Only the rate and samples fields of capture_processing_format_ are used 353 // because the capture processing number of channels is mutable and is 354 // tracked by the capture_audio_. 355 StreamConfig capture_processing_format; 356 int split_rate; 357 bool echo_path_gain_change; 358 } capture_ RTC_GUARDED_BY(crit_capture_); 359 360 struct ApmCaptureNonLockedState { ApmCaptureNonLockedStateApmCaptureNonLockedState361 ApmCaptureNonLockedState(bool beamformer_enabled, 362 bool intelligibility_enabled) 363 : capture_processing_format(kSampleRate16kHz), 364 split_rate(kSampleRate16kHz), 365 stream_delay_ms(0), 366 beamformer_enabled(beamformer_enabled), 367 intelligibility_enabled(intelligibility_enabled) {} 368 // Only the rate and samples fields of capture_processing_format_ are used 369 // because the forward processing number of channels is mutable and is 370 // tracked by the capture_audio_. 371 StreamConfig capture_processing_format; 372 int split_rate; 373 int stream_delay_ms; 374 bool beamformer_enabled; 375 bool intelligibility_enabled; 376 bool level_controller_enabled = false; 377 bool echo_controller_enabled = false; 378 } capture_nonlocked_; 379 380 struct ApmRenderState { 381 ApmRenderState(); 382 ~ApmRenderState(); 383 std::unique_ptr<AudioConverter> render_converter; 384 std::unique_ptr<AudioBuffer> render_audio; 385 } render_ RTC_GUARDED_BY(crit_render_); 386 387 size_t aec_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) 388 RTC_GUARDED_BY(crit_capture_) = 0; 389 std::vector<float> aec_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); 390 std::vector<float> aec_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); 391 392 size_t aecm_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) 393 RTC_GUARDED_BY(crit_capture_) = 0; 394 std::vector<int16_t> aecm_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); 395 std::vector<int16_t> aecm_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); 396 397 size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) 398 RTC_GUARDED_BY(crit_capture_) = 0; 399 std::vector<int16_t> agc_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); 400 std::vector<int16_t> agc_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); 401 402 size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_) 403 RTC_GUARDED_BY(crit_capture_) = 0; 404 std::vector<float> red_render_queue_buffer_ RTC_GUARDED_BY(crit_render_); 405 std::vector<float> red_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_); 406 407 RmsLevel capture_input_rms_ RTC_GUARDED_BY(crit_capture_); 408 RmsLevel capture_output_rms_ RTC_GUARDED_BY(crit_capture_); 409 int capture_rms_interval_counter_ RTC_GUARDED_BY(crit_capture_) = 0; 410 411 // Lock protection not needed. 412 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>> 413 aec_render_signal_queue_; 414 std::unique_ptr< 415 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>> 416 aecm_render_signal_queue_; 417 std::unique_ptr< 418 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>> 419 agc_render_signal_queue_; 420 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>> 421 red_render_signal_queue_; 422 }; 423 424 } // namespace webrtc 425 426 #endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_ 427