1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_processing/audio_processing_impl.h"
12
13 #include <math.h>
14 #include <algorithm>
15 #include <string>
16
17 #include "common_audio/audio_converter.h"
18 #include "common_audio/channel_buffer.h"
19 #include "common_audio/include/audio_util.h"
20 #include "common_audio/signal_processing/include/signal_processing_library.h"
21 #include "modules/audio_processing/aec/aec_core.h"
22 #include "modules/audio_processing/aec3/echo_canceller3.h"
23 #include "modules/audio_processing/agc/agc_manager_direct.h"
24 #include "modules/audio_processing/agc2/gain_controller2.h"
25 #include "modules/audio_processing/audio_buffer.h"
26 #include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
27 #include "modules/audio_processing/common.h"
28 #include "modules/audio_processing/echo_cancellation_impl.h"
29 #include "modules/audio_processing/echo_control_mobile_impl.h"
30 #include "modules/audio_processing/gain_control_for_experimental_agc.h"
31 #include "modules/audio_processing/gain_control_impl.h"
32 #include "rtc_base/checks.h"
33 #include "rtc_base/logging.h"
34 #include "rtc_base/platform_file.h"
35 #include "rtc_base/refcountedobject.h"
36 #include "rtc_base/trace_event.h"
37 #if WEBRTC_INTELLIGIBILITY_ENHANCER
38 #include "modules/audio_processing/intelligibility/intelligibility_enhancer.h"
39 #endif
40 #include "modules/audio_processing/level_controller/level_controller.h"
41 #include "modules/audio_processing/level_estimator_impl.h"
42 #include "modules/audio_processing/low_cut_filter.h"
43 #include "modules/audio_processing/noise_suppression_impl.h"
44 #include "modules/audio_processing/residual_echo_detector.h"
45 #include "modules/audio_processing/transient/transient_suppressor.h"
46 #include "modules/audio_processing/voice_detection_impl.h"
47 #include "modules/include/module_common_types.h"
48 #include "system_wrappers/include/file_wrapper.h"
49 #include "system_wrappers/include/metrics.h"
50
51 // Check to verify that the define for the intelligibility enhancer is properly
52 // set.
53 #if !defined(WEBRTC_INTELLIGIBILITY_ENHANCER) || \
54 (WEBRTC_INTELLIGIBILITY_ENHANCER != 0 && \
55 WEBRTC_INTELLIGIBILITY_ENHANCER != 1)
56 #error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1"
57 #endif
58
59 #define RETURN_ON_ERR(expr) \
60 do { \
61 int err = (expr); \
62 if (err != kNoError) { \
63 return err; \
64 } \
65 } while (0)
66
67 namespace webrtc {
68
69 constexpr int AudioProcessing::kNativeSampleRatesHz[];
70
71 namespace {
72
LayoutHasKeyboard(AudioProcessing::ChannelLayout layout)73 static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
74 switch (layout) {
75 case AudioProcessing::kMono:
76 case AudioProcessing::kStereo:
77 return false;
78 case AudioProcessing::kMonoAndKeyboard:
79 case AudioProcessing::kStereoAndKeyboard:
80 return true;
81 }
82
83 RTC_NOTREACHED();
84 return false;
85 }
86
SampleRateSupportsMultiBand(int sample_rate_hz)87 bool SampleRateSupportsMultiBand(int sample_rate_hz) {
88 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
89 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
90 }
91
FindNativeProcessRateToUse(int minimum_rate,bool band_splitting_required)92 int FindNativeProcessRateToUse(int minimum_rate, bool band_splitting_required) {
93 #ifdef WEBRTC_ARCH_ARM_FAMILY
94 constexpr int kMaxSplittingNativeProcessRate =
95 AudioProcessing::kSampleRate32kHz;
96 #else
97 constexpr int kMaxSplittingNativeProcessRate =
98 AudioProcessing::kSampleRate48kHz;
99 #endif
100 static_assert(
101 kMaxSplittingNativeProcessRate <= AudioProcessing::kMaxNativeSampleRateHz,
102 "");
103 const int uppermost_native_rate = band_splitting_required
104 ? kMaxSplittingNativeProcessRate
105 : AudioProcessing::kSampleRate48kHz;
106
107 for (auto rate : AudioProcessing::kNativeSampleRatesHz) {
108 if (rate >= uppermost_native_rate) {
109 return uppermost_native_rate;
110 }
111 if (rate >= minimum_rate) {
112 return rate;
113 }
114 }
115 RTC_NOTREACHED();
116 return uppermost_native_rate;
117 }
118
119 // Maximum lengths that frame of samples being passed from the render side to
120 // the capture side can have (does not apply to AEC3).
121 static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
122 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
123
124 // Maximum number of frames to buffer in the render queue.
125 // TODO(peah): Decrease this once we properly handle hugely unbalanced
126 // reverse and forward call numbers.
127 static const size_t kMaxNumFramesToBuffer = 100;
128
129 class HighPassFilterImpl : public HighPassFilter {
130 public:
HighPassFilterImpl(AudioProcessingImpl * apm)131 explicit HighPassFilterImpl(AudioProcessingImpl* apm) : apm_(apm) {}
132 ~HighPassFilterImpl() override = default;
133
134 // HighPassFilter implementation.
Enable(bool enable)135 int Enable(bool enable) override {
136 apm_->MutateConfig([enable](AudioProcessing::Config* config) {
137 config->high_pass_filter.enabled = enable;
138 });
139
140 return AudioProcessing::kNoError;
141 }
142
is_enabled() const143 bool is_enabled() const override {
144 return apm_->GetConfig().high_pass_filter.enabled;
145 }
146
147 private:
148 AudioProcessingImpl* apm_;
149 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(HighPassFilterImpl);
150 };
151
ToStreamsConfig(const ProcessingConfig & api_format)152 webrtc::InternalAPMStreamsConfig ToStreamsConfig(
153 const ProcessingConfig& api_format) {
154 webrtc::InternalAPMStreamsConfig result;
155 result.input_sample_rate = api_format.input_stream().sample_rate_hz();
156 result.input_num_channels = api_format.input_stream().num_channels();
157 result.output_num_channels = api_format.output_stream().num_channels();
158 result.render_input_num_channels =
159 api_format.reverse_input_stream().num_channels();
160 result.render_input_sample_rate =
161 api_format.reverse_input_stream().sample_rate_hz();
162 result.output_sample_rate = api_format.output_stream().sample_rate_hz();
163 result.render_output_sample_rate =
164 api_format.reverse_output_stream().sample_rate_hz();
165 result.render_output_num_channels =
166 api_format.reverse_output_stream().num_channels();
167 return result;
168 }
169 } // namespace
170
171 // Throughout webrtc, it's assumed that success is represented by zero.
172 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
173
ApmSubmoduleStates(bool capture_post_processor_enabled)174 AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates(
175 bool capture_post_processor_enabled)
176 : capture_post_processor_enabled_(capture_post_processor_enabled) {}
177
Update(bool low_cut_filter_enabled,bool echo_canceller_enabled,bool mobile_echo_controller_enabled,bool residual_echo_detector_enabled,bool noise_suppressor_enabled,bool intelligibility_enhancer_enabled,bool beamformer_enabled,bool adaptive_gain_controller_enabled,bool gain_controller2_enabled,bool level_controller_enabled,bool echo_controller_enabled,bool voice_activity_detector_enabled,bool level_estimator_enabled,bool transient_suppressor_enabled)178 bool AudioProcessingImpl::ApmSubmoduleStates::Update(
179 bool low_cut_filter_enabled,
180 bool echo_canceller_enabled,
181 bool mobile_echo_controller_enabled,
182 bool residual_echo_detector_enabled,
183 bool noise_suppressor_enabled,
184 bool intelligibility_enhancer_enabled,
185 bool beamformer_enabled,
186 bool adaptive_gain_controller_enabled,
187 bool gain_controller2_enabled,
188 bool level_controller_enabled,
189 bool echo_controller_enabled,
190 bool voice_activity_detector_enabled,
191 bool level_estimator_enabled,
192 bool transient_suppressor_enabled) {
193 bool changed = false;
194 changed |= (low_cut_filter_enabled != low_cut_filter_enabled_);
195 changed |= (echo_canceller_enabled != echo_canceller_enabled_);
196 changed |=
197 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
198 changed |=
199 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
200 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
201 changed |=
202 (intelligibility_enhancer_enabled != intelligibility_enhancer_enabled_);
203 changed |= (beamformer_enabled != beamformer_enabled_);
204 changed |=
205 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
206 changed |=
207 (gain_controller2_enabled != gain_controller2_enabled_);
208 changed |= (level_controller_enabled != level_controller_enabled_);
209 changed |= (echo_controller_enabled != echo_controller_enabled_);
210 changed |= (level_estimator_enabled != level_estimator_enabled_);
211 changed |=
212 (voice_activity_detector_enabled != voice_activity_detector_enabled_);
213 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
214 if (changed) {
215 low_cut_filter_enabled_ = low_cut_filter_enabled;
216 echo_canceller_enabled_ = echo_canceller_enabled;
217 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
218 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
219 noise_suppressor_enabled_ = noise_suppressor_enabled;
220 intelligibility_enhancer_enabled_ = intelligibility_enhancer_enabled;
221 beamformer_enabled_ = beamformer_enabled;
222 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
223 gain_controller2_enabled_ = gain_controller2_enabled;
224 level_controller_enabled_ = level_controller_enabled;
225 echo_controller_enabled_ = echo_controller_enabled;
226 level_estimator_enabled_ = level_estimator_enabled;
227 voice_activity_detector_enabled_ = voice_activity_detector_enabled;
228 transient_suppressor_enabled_ = transient_suppressor_enabled;
229 }
230
231 changed |= first_update_;
232 first_update_ = false;
233 return changed;
234 }
235
CaptureMultiBandSubModulesActive() const236 bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandSubModulesActive()
237 const {
238 #if WEBRTC_INTELLIGIBILITY_ENHANCER
239 return CaptureMultiBandProcessingActive() ||
240 intelligibility_enhancer_enabled_ || voice_activity_detector_enabled_;
241 #else
242 return CaptureMultiBandProcessingActive() || voice_activity_detector_enabled_;
243 #endif
244 }
245
CaptureMultiBandProcessingActive() const246 bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandProcessingActive()
247 const {
248 return low_cut_filter_enabled_ || echo_canceller_enabled_ ||
249 mobile_echo_controller_enabled_ || noise_suppressor_enabled_ ||
250 beamformer_enabled_ || adaptive_gain_controller_enabled_ ||
251 echo_controller_enabled_;
252 }
253
CaptureFullBandProcessingActive() const254 bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive()
255 const {
256 return level_controller_enabled_ || gain_controller2_enabled_ ||
257 capture_post_processor_enabled_;
258 }
259
RenderMultiBandSubModulesActive() const260 bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
261 const {
262 return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
263 mobile_echo_controller_enabled_ || adaptive_gain_controller_enabled_ ||
264 echo_controller_enabled_;
265 }
266
RenderMultiBandProcessingActive() const267 bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandProcessingActive()
268 const {
269 #if WEBRTC_INTELLIGIBILITY_ENHANCER
270 return intelligibility_enhancer_enabled_;
271 #else
272 return false;
273 #endif
274 }
275
276 struct AudioProcessingImpl::ApmPublicSubmodules {
ApmPublicSubmoduleswebrtc::AudioProcessingImpl::ApmPublicSubmodules277 ApmPublicSubmodules() {}
278 // Accessed externally of APM without any lock acquired.
279 std::unique_ptr<EchoCancellationImpl> echo_cancellation;
280 std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
281 std::unique_ptr<GainControlImpl> gain_control;
282 std::unique_ptr<LevelEstimatorImpl> level_estimator;
283 std::unique_ptr<NoiseSuppressionImpl> noise_suppression;
284 std::unique_ptr<VoiceDetectionImpl> voice_detection;
285 std::unique_ptr<GainControlForExperimentalAgc>
286 gain_control_for_experimental_agc;
287
288 // Accessed internally from both render and capture.
289 std::unique_ptr<TransientSuppressor> transient_suppressor;
290 #if WEBRTC_INTELLIGIBILITY_ENHANCER
291 std::unique_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
292 #endif
293 };
294
295 struct AudioProcessingImpl::ApmPrivateSubmodules {
ApmPrivateSubmoduleswebrtc::AudioProcessingImpl::ApmPrivateSubmodules296 ApmPrivateSubmodules(NonlinearBeamformer* beamformer,
297 std::unique_ptr<PostProcessing> capture_post_processor)
298 : beamformer(beamformer),
299 capture_post_processor(std::move(capture_post_processor)) {}
300 // Accessed internally from capture or during initialization
301 std::unique_ptr<NonlinearBeamformer> beamformer;
302 std::unique_ptr<AgcManagerDirect> agc_manager;
303 std::unique_ptr<GainController2> gain_controller2;
304 std::unique_ptr<LowCutFilter> low_cut_filter;
305 std::unique_ptr<LevelController> level_controller;
306 std::unique_ptr<ResidualEchoDetector> residual_echo_detector;
307 std::unique_ptr<EchoControl> echo_controller;
308 std::unique_ptr<PostProcessing> capture_post_processor;
309 };
310
Create()311 AudioProcessing* AudioProcessing::Create() {
312 webrtc::Config config;
313 return Create(config, nullptr, nullptr, nullptr);
314 }
315
Create(const webrtc::Config & config)316 AudioProcessing* AudioProcessing::Create(const webrtc::Config& config) {
317 return Create(config, nullptr, nullptr, nullptr);
318 }
319
Create(const webrtc::Config & config,NonlinearBeamformer * beamformer)320 AudioProcessing* AudioProcessing::Create(const webrtc::Config& config,
321 NonlinearBeamformer* beamformer) {
322 return Create(config, nullptr, nullptr, beamformer);
323 }
324
Create(const webrtc::Config & config,std::unique_ptr<PostProcessing> capture_post_processor,std::unique_ptr<EchoControlFactory> echo_control_factory,NonlinearBeamformer * beamformer)325 AudioProcessing* AudioProcessing::Create(
326 const webrtc::Config& config,
327 std::unique_ptr<PostProcessing> capture_post_processor,
328 std::unique_ptr<EchoControlFactory> echo_control_factory,
329 NonlinearBeamformer* beamformer) {
330 AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
331 config, std::move(capture_post_processor),
332 std::move(echo_control_factory), beamformer);
333 if (apm->Initialize() != kNoError) {
334 delete apm;
335 apm = nullptr;
336 }
337
338 return apm;
339 }
340
AudioProcessingImpl(const webrtc::Config & config)341 AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
342 : AudioProcessingImpl(config, nullptr, nullptr, nullptr) {}
343
AudioProcessingImpl(const webrtc::Config & config,std::unique_ptr<PostProcessing> capture_post_processor,std::unique_ptr<EchoControlFactory> echo_control_factory,NonlinearBeamformer * beamformer)344 AudioProcessingImpl::AudioProcessingImpl(
345 const webrtc::Config& config,
346 std::unique_ptr<PostProcessing> capture_post_processor,
347 std::unique_ptr<EchoControlFactory> echo_control_factory,
348 NonlinearBeamformer* beamformer)
349 : high_pass_filter_impl_(new HighPassFilterImpl(this)),
350 echo_control_factory_(std::move(echo_control_factory)),
351 submodule_states_(!!capture_post_processor),
352 public_submodules_(new ApmPublicSubmodules()),
353 private_submodules_(
354 new ApmPrivateSubmodules(beamformer,
355 std::move(capture_post_processor))),
356 constants_(config.Get<ExperimentalAgc>().startup_min_volume,
357 config.Get<ExperimentalAgc>().clipped_level_min,
358 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
359 false),
360 #else
361 config.Get<ExperimentalAgc>().enabled),
362 #endif
363 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
364 capture_(false,
365 #else
366 capture_(config.Get<ExperimentalNs>().enabled,
367 #endif
368 config.Get<Beamforming>().array_geometry,
369 config.Get<Beamforming>().target_direction),
370 capture_nonlocked_(config.Get<Beamforming>().enabled,
371 config.Get<Intelligibility>().enabled) {
372 {
373 rtc::CritScope cs_render(&crit_render_);
374 rtc::CritScope cs_capture(&crit_capture_);
375
376 // Mark Echo Controller enabled if a factory is injected.
377 capture_nonlocked_.echo_controller_enabled =
378 static_cast<bool>(echo_control_factory_);
379
380 public_submodules_->echo_cancellation.reset(
381 new EchoCancellationImpl(&crit_render_, &crit_capture_));
382 public_submodules_->echo_control_mobile.reset(
383 new EchoControlMobileImpl(&crit_render_, &crit_capture_));
384 public_submodules_->gain_control.reset(
385 new GainControlImpl(&crit_capture_, &crit_capture_));
386 public_submodules_->level_estimator.reset(
387 new LevelEstimatorImpl(&crit_capture_));
388 public_submodules_->noise_suppression.reset(
389 new NoiseSuppressionImpl(&crit_capture_));
390 public_submodules_->voice_detection.reset(
391 new VoiceDetectionImpl(&crit_capture_));
392 public_submodules_->gain_control_for_experimental_agc.reset(
393 new GainControlForExperimentalAgc(
394 public_submodules_->gain_control.get(), &crit_capture_));
395 private_submodules_->residual_echo_detector.reset(
396 new ResidualEchoDetector());
397
398 // TODO(peah): Move this creation to happen only when the level controller
399 // is enabled.
400 private_submodules_->level_controller.reset(new LevelController());
401
402 // TODO(alessiob): Move the injected gain controller once injection is
403 // implemented.
404 private_submodules_->gain_controller2.reset(new GainController2());
405
406 RTC_LOG(LS_INFO) << "Capture post processor activated: "
407 << !!private_submodules_->capture_post_processor;
408 }
409
410 SetExtraOptions(config);
411 }
412
~AudioProcessingImpl()413 AudioProcessingImpl::~AudioProcessingImpl() {
414 // Depends on gain_control_ and
415 // public_submodules_->gain_control_for_experimental_agc.
416 private_submodules_->agc_manager.reset();
417 // Depends on gain_control_.
418 public_submodules_->gain_control_for_experimental_agc.reset();
419 }
420
Initialize()421 int AudioProcessingImpl::Initialize() {
422 // Run in a single-threaded manner during initialization.
423 rtc::CritScope cs_render(&crit_render_);
424 rtc::CritScope cs_capture(&crit_capture_);
425 return InitializeLocked();
426 }
427
Initialize(int capture_input_sample_rate_hz,int capture_output_sample_rate_hz,int render_input_sample_rate_hz,ChannelLayout capture_input_layout,ChannelLayout capture_output_layout,ChannelLayout render_input_layout)428 int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
429 int capture_output_sample_rate_hz,
430 int render_input_sample_rate_hz,
431 ChannelLayout capture_input_layout,
432 ChannelLayout capture_output_layout,
433 ChannelLayout render_input_layout) {
434 const ProcessingConfig processing_config = {
435 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
436 LayoutHasKeyboard(capture_input_layout)},
437 {capture_output_sample_rate_hz,
438 ChannelsFromLayout(capture_output_layout),
439 LayoutHasKeyboard(capture_output_layout)},
440 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
441 LayoutHasKeyboard(render_input_layout)},
442 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
443 LayoutHasKeyboard(render_input_layout)}}};
444
445 return Initialize(processing_config);
446 }
447
Initialize(const ProcessingConfig & processing_config)448 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
449 // Run in a single-threaded manner during initialization.
450 rtc::CritScope cs_render(&crit_render_);
451 rtc::CritScope cs_capture(&crit_capture_);
452 return InitializeLocked(processing_config);
453 }
454
MaybeInitializeRender(const ProcessingConfig & processing_config)455 int AudioProcessingImpl::MaybeInitializeRender(
456 const ProcessingConfig& processing_config) {
457 return MaybeInitialize(processing_config, false);
458 }
459
MaybeInitializeCapture(const ProcessingConfig & processing_config,bool force_initialization)460 int AudioProcessingImpl::MaybeInitializeCapture(
461 const ProcessingConfig& processing_config,
462 bool force_initialization) {
463 return MaybeInitialize(processing_config, force_initialization);
464 }
465
466 // Calls InitializeLocked() if any of the audio parameters have changed from
467 // their current values (needs to be called while holding the crit_render_lock).
MaybeInitialize(const ProcessingConfig & processing_config,bool force_initialization)468 int AudioProcessingImpl::MaybeInitialize(
469 const ProcessingConfig& processing_config,
470 bool force_initialization) {
471 // Called from both threads. Thread check is therefore not possible.
472 if (processing_config == formats_.api_format && !force_initialization) {
473 return kNoError;
474 }
475
476 rtc::CritScope cs_capture(&crit_capture_);
477 return InitializeLocked(processing_config);
478 }
479
InitializeLocked()480 int AudioProcessingImpl::InitializeLocked() {
481 UpdateActiveSubmoduleStates();
482
483 const int capture_audiobuffer_num_channels =
484 capture_nonlocked_.beamformer_enabled
485 ? formats_.api_format.input_stream().num_channels()
486 : formats_.api_format.output_stream().num_channels();
487
488 const int render_audiobuffer_num_output_frames =
489 formats_.api_format.reverse_output_stream().num_frames() == 0
490 ? formats_.render_processing_format.num_frames()
491 : formats_.api_format.reverse_output_stream().num_frames();
492 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
493 render_.render_audio.reset(new AudioBuffer(
494 formats_.api_format.reverse_input_stream().num_frames(),
495 formats_.api_format.reverse_input_stream().num_channels(),
496 formats_.render_processing_format.num_frames(),
497 formats_.render_processing_format.num_channels(),
498 render_audiobuffer_num_output_frames));
499 if (formats_.api_format.reverse_input_stream() !=
500 formats_.api_format.reverse_output_stream()) {
501 render_.render_converter = AudioConverter::Create(
502 formats_.api_format.reverse_input_stream().num_channels(),
503 formats_.api_format.reverse_input_stream().num_frames(),
504 formats_.api_format.reverse_output_stream().num_channels(),
505 formats_.api_format.reverse_output_stream().num_frames());
506 } else {
507 render_.render_converter.reset(nullptr);
508 }
509 } else {
510 render_.render_audio.reset(nullptr);
511 render_.render_converter.reset(nullptr);
512 }
513
514 capture_.capture_audio.reset(
515 new AudioBuffer(formats_.api_format.input_stream().num_frames(),
516 formats_.api_format.input_stream().num_channels(),
517 capture_nonlocked_.capture_processing_format.num_frames(),
518 capture_audiobuffer_num_channels,
519 formats_.api_format.output_stream().num_frames()));
520
521 public_submodules_->echo_cancellation->Initialize(
522 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
523 num_proc_channels());
524 AllocateRenderQueue();
525
526 int success = public_submodules_->echo_cancellation->enable_metrics(true);
527 RTC_DCHECK_EQ(0, success);
528 success = public_submodules_->echo_cancellation->enable_delay_logging(true);
529 RTC_DCHECK_EQ(0, success);
530 public_submodules_->echo_control_mobile->Initialize(
531 proc_split_sample_rate_hz(), num_reverse_channels(),
532 num_output_channels());
533
534 public_submodules_->gain_control->Initialize(num_proc_channels(),
535 proc_sample_rate_hz());
536 if (constants_.use_experimental_agc) {
537 if (!private_submodules_->agc_manager.get()) {
538 private_submodules_->agc_manager.reset(new AgcManagerDirect(
539 public_submodules_->gain_control.get(),
540 public_submodules_->gain_control_for_experimental_agc.get(),
541 constants_.agc_startup_min_volume, constants_.agc_clipped_level_min));
542 }
543 private_submodules_->agc_manager->Initialize();
544 private_submodules_->agc_manager->SetCaptureMuted(
545 capture_.output_will_be_muted);
546 public_submodules_->gain_control_for_experimental_agc->Initialize();
547 }
548 InitializeTransient();
549 InitializeBeamformer();
550 #if WEBRTC_INTELLIGIBILITY_ENHANCER
551 InitializeIntelligibility();
552 #endif
553 InitializeLowCutFilter();
554 public_submodules_->noise_suppression->Initialize(num_proc_channels(),
555 proc_sample_rate_hz());
556 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz());
557 public_submodules_->level_estimator->Initialize();
558 InitializeLevelController();
559 InitializeResidualEchoDetector();
560 InitializeEchoController();
561 InitializeGainController2();
562 InitializePostProcessor();
563
564 if (aec_dump_) {
565 aec_dump_->WriteInitMessage(ToStreamsConfig(formats_.api_format));
566 }
567 return kNoError;
568 }
569
InitializeLocked(const ProcessingConfig & config)570 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
571 UpdateActiveSubmoduleStates();
572
573 for (const auto& stream : config.streams) {
574 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
575 return kBadSampleRateError;
576 }
577 }
578
579 const size_t num_in_channels = config.input_stream().num_channels();
580 const size_t num_out_channels = config.output_stream().num_channels();
581
582 // Need at least one input channel.
583 // Need either one output channel or as many outputs as there are inputs.
584 if (num_in_channels == 0 ||
585 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
586 return kBadNumberChannelsError;
587 }
588
589 if (capture_nonlocked_.beamformer_enabled &&
590 num_in_channels != capture_.array_geometry.size()) {
591 return kBadNumberChannelsError;
592 }
593
594 formats_.api_format = config;
595
596 int capture_processing_rate = FindNativeProcessRateToUse(
597 std::min(formats_.api_format.input_stream().sample_rate_hz(),
598 formats_.api_format.output_stream().sample_rate_hz()),
599 submodule_states_.CaptureMultiBandSubModulesActive() ||
600 submodule_states_.RenderMultiBandSubModulesActive());
601
602 capture_nonlocked_.capture_processing_format =
603 StreamConfig(capture_processing_rate);
604
605 int render_processing_rate;
606 if (!capture_nonlocked_.echo_controller_enabled) {
607 render_processing_rate = FindNativeProcessRateToUse(
608 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
609 formats_.api_format.reverse_output_stream().sample_rate_hz()),
610 submodule_states_.CaptureMultiBandSubModulesActive() ||
611 submodule_states_.RenderMultiBandSubModulesActive());
612 } else {
613 render_processing_rate = capture_processing_rate;
614 }
615
616 // TODO(aluebs): Remove this restriction once we figure out why the 3-band
617 // splitting filter degrades the AEC performance.
618 if (render_processing_rate > kSampleRate32kHz &&
619 !capture_nonlocked_.echo_controller_enabled) {
620 render_processing_rate = submodule_states_.RenderMultiBandProcessingActive()
621 ? kSampleRate32kHz
622 : kSampleRate16kHz;
623 }
624
625 // If the forward sample rate is 8 kHz, the render stream is also processed
626 // at this rate.
627 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
628 kSampleRate8kHz) {
629 render_processing_rate = kSampleRate8kHz;
630 } else {
631 render_processing_rate =
632 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
633 }
634
635 // Always downmix the render stream to mono for analysis. This has been
636 // demonstrated to work well for AEC in most practical scenarios.
637 if (submodule_states_.RenderMultiBandSubModulesActive()) {
638 formats_.render_processing_format = StreamConfig(render_processing_rate, 1);
639 } else {
640 formats_.render_processing_format = StreamConfig(
641 formats_.api_format.reverse_input_stream().sample_rate_hz(),
642 formats_.api_format.reverse_input_stream().num_channels());
643 }
644
645 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
646 kSampleRate32kHz ||
647 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
648 kSampleRate48kHz) {
649 capture_nonlocked_.split_rate = kSampleRate16kHz;
650 } else {
651 capture_nonlocked_.split_rate =
652 capture_nonlocked_.capture_processing_format.sample_rate_hz();
653 }
654
655 return InitializeLocked();
656 }
657
ApplyConfig(const AudioProcessing::Config & config)658 void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
659 config_ = config;
660
661 bool config_ok = LevelController::Validate(config_.level_controller);
662 if (!config_ok) {
663 RTC_LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl
664 << "level_controller: "
665 << LevelController::ToString(config_.level_controller)
666 << std::endl
667 << "Reverting to default parameter set";
668 config_.level_controller = AudioProcessing::Config::LevelController();
669 }
670
671 // Run in a single-threaded manner when applying the settings.
672 rtc::CritScope cs_render(&crit_render_);
673 rtc::CritScope cs_capture(&crit_capture_);
674
675 // TODO(peah): Replace the use of capture_nonlocked_.level_controller_enabled
676 // with the value in config_ everywhere in the code.
677 if (capture_nonlocked_.level_controller_enabled !=
678 config_.level_controller.enabled) {
679 capture_nonlocked_.level_controller_enabled =
680 config_.level_controller.enabled;
681 // TODO(peah): Remove the conditional initialization to always initialize
682 // the level controller regardless of whether it is enabled or not.
683 InitializeLevelController();
684 }
685 RTC_LOG(LS_INFO) << "Level controller activated: "
686 << capture_nonlocked_.level_controller_enabled;
687
688 private_submodules_->level_controller->ApplyConfig(config_.level_controller);
689
690 InitializeLowCutFilter();
691
692 RTC_LOG(LS_INFO) << "Highpass filter activated: "
693 << config_.high_pass_filter.enabled;
694
695 // Deprecated way of activating AEC3.
696 // TODO(gustaf): Remove when possible.
697 if (config.echo_canceller3.enabled && !echo_control_factory_) {
698 capture_nonlocked_.echo_controller_enabled =
699 config_.echo_canceller3.enabled;
700 echo_control_factory_ =
701 std::unique_ptr<EchoControlFactory>(new EchoCanceller3Factory());
702 InitializeEchoController();
703 RTC_LOG(LS_INFO) << "Echo canceller 3 activated: "
704 << capture_nonlocked_.echo_controller_enabled;
705 }
706
707 config_ok = GainController2::Validate(config_.gain_controller2);
708 if (!config_ok) {
709 RTC_LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl
710 << "Gain Controller 2: "
711 << GainController2::ToString(config_.gain_controller2)
712 << std::endl
713 << "Reverting to default parameter set";
714 config_.gain_controller2 = AudioProcessing::Config::GainController2();
715 }
716 InitializeGainController2();
717 private_submodules_->gain_controller2->ApplyConfig(config_.gain_controller2);
718 RTC_LOG(LS_INFO) << "Gain Controller 2 activated: "
719 << config_.gain_controller2.enabled;
720 }
721
SetExtraOptions(const webrtc::Config & config)722 void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
723 // Run in a single-threaded manner when setting the extra options.
724 rtc::CritScope cs_render(&crit_render_);
725 rtc::CritScope cs_capture(&crit_capture_);
726
727 public_submodules_->echo_cancellation->SetExtraOptions(config);
728
729 if (capture_.transient_suppressor_enabled !=
730 config.Get<ExperimentalNs>().enabled) {
731 capture_.transient_suppressor_enabled =
732 config.Get<ExperimentalNs>().enabled;
733 InitializeTransient();
734 }
735
736 #if WEBRTC_INTELLIGIBILITY_ENHANCER
737 if (capture_nonlocked_.intelligibility_enabled !=
738 config.Get<Intelligibility>().enabled) {
739 capture_nonlocked_.intelligibility_enabled =
740 config.Get<Intelligibility>().enabled;
741 InitializeIntelligibility();
742 }
743 #endif
744
745 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
746 if (capture_nonlocked_.beamformer_enabled !=
747 config.Get<Beamforming>().enabled) {
748 capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
749 if (config.Get<Beamforming>().array_geometry.size() > 1) {
750 capture_.array_geometry = config.Get<Beamforming>().array_geometry;
751 }
752 capture_.target_direction = config.Get<Beamforming>().target_direction;
753 InitializeBeamformer();
754 }
755 #endif // WEBRTC_ANDROID_PLATFORM_BUILD
756 }
757
proc_sample_rate_hz() const758 int AudioProcessingImpl::proc_sample_rate_hz() const {
759 // Used as callback from submodules, hence locking is not allowed.
760 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
761 }
762
proc_split_sample_rate_hz() const763 int AudioProcessingImpl::proc_split_sample_rate_hz() const {
764 // Used as callback from submodules, hence locking is not allowed.
765 return capture_nonlocked_.split_rate;
766 }
767
num_reverse_channels() const768 size_t AudioProcessingImpl::num_reverse_channels() const {
769 // Used as callback from submodules, hence locking is not allowed.
770 return formats_.render_processing_format.num_channels();
771 }
772
num_input_channels() const773 size_t AudioProcessingImpl::num_input_channels() const {
774 // Used as callback from submodules, hence locking is not allowed.
775 return formats_.api_format.input_stream().num_channels();
776 }
777
num_proc_channels() const778 size_t AudioProcessingImpl::num_proc_channels() const {
779 // Used as callback from submodules, hence locking is not allowed.
780 return (capture_nonlocked_.beamformer_enabled ||
781 capture_nonlocked_.echo_controller_enabled)
782 ? 1
783 : num_output_channels();
784 }
785
num_output_channels() const786 size_t AudioProcessingImpl::num_output_channels() const {
787 // Used as callback from submodules, hence locking is not allowed.
788 return formats_.api_format.output_stream().num_channels();
789 }
790
set_output_will_be_muted(bool muted)791 void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
792 rtc::CritScope cs(&crit_capture_);
793 capture_.output_will_be_muted = muted;
794 if (private_submodules_->agc_manager.get()) {
795 private_submodules_->agc_manager->SetCaptureMuted(
796 capture_.output_will_be_muted);
797 }
798 }
799
800
ProcessStream(const float * const * src,size_t samples_per_channel,int input_sample_rate_hz,ChannelLayout input_layout,int output_sample_rate_hz,ChannelLayout output_layout,float * const * dest)801 int AudioProcessingImpl::ProcessStream(const float* const* src,
802 size_t samples_per_channel,
803 int input_sample_rate_hz,
804 ChannelLayout input_layout,
805 int output_sample_rate_hz,
806 ChannelLayout output_layout,
807 float* const* dest) {
808 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_ChannelLayout");
809 StreamConfig input_stream;
810 StreamConfig output_stream;
811 {
812 // Access the formats_.api_format.input_stream beneath the capture lock.
813 // The lock must be released as it is later required in the call
814 // to ProcessStream(,,,);
815 rtc::CritScope cs(&crit_capture_);
816 input_stream = formats_.api_format.input_stream();
817 output_stream = formats_.api_format.output_stream();
818 }
819
820 input_stream.set_sample_rate_hz(input_sample_rate_hz);
821 input_stream.set_num_channels(ChannelsFromLayout(input_layout));
822 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout));
823 output_stream.set_sample_rate_hz(output_sample_rate_hz);
824 output_stream.set_num_channels(ChannelsFromLayout(output_layout));
825 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout));
826
827 if (samples_per_channel != input_stream.num_frames()) {
828 return kBadDataLengthError;
829 }
830 return ProcessStream(src, input_stream, output_stream, dest);
831 }
832
ProcessStream(const float * const * src,const StreamConfig & input_config,const StreamConfig & output_config,float * const * dest)833 int AudioProcessingImpl::ProcessStream(const float* const* src,
834 const StreamConfig& input_config,
835 const StreamConfig& output_config,
836 float* const* dest) {
837 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
838 ProcessingConfig processing_config;
839 bool reinitialization_required = false;
840 {
841 // Acquire the capture lock in order to safely call the function
842 // that retrieves the render side data. This function accesses apm
843 // getters that need the capture lock held when being called.
844 rtc::CritScope cs_capture(&crit_capture_);
845 EmptyQueuedRenderAudio();
846
847 if (!src || !dest) {
848 return kNullPointerError;
849 }
850
851 processing_config = formats_.api_format;
852 reinitialization_required = UpdateActiveSubmoduleStates();
853 }
854
855 processing_config.input_stream() = input_config;
856 processing_config.output_stream() = output_config;
857
858 {
859 // Do conditional reinitialization.
860 rtc::CritScope cs_render(&crit_render_);
861 RETURN_ON_ERR(
862 MaybeInitializeCapture(processing_config, reinitialization_required));
863 }
864 rtc::CritScope cs_capture(&crit_capture_);
865 RTC_DCHECK_EQ(processing_config.input_stream().num_frames(),
866 formats_.api_format.input_stream().num_frames());
867
868 if (aec_dump_) {
869 RecordUnprocessedCaptureStream(src);
870 }
871
872 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
873 RETURN_ON_ERR(ProcessCaptureStreamLocked());
874 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
875
876 if (aec_dump_) {
877 RecordProcessedCaptureStream(dest);
878 }
879 return kNoError;
880 }
881
QueueBandedRenderAudio(AudioBuffer * audio)882 void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
883 EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
884 num_reverse_channels(),
885 &aec_render_queue_buffer_);
886
887 RTC_DCHECK_GE(160, audio->num_frames_per_band());
888
889 // Insert the samples into the queue.
890 if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
891 // The data queue is full and needs to be emptied.
892 EmptyQueuedRenderAudio();
893
894 // Retry the insert (should always work).
895 bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
896 RTC_DCHECK(result);
897 }
898
899 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
900 num_reverse_channels(),
901 &aecm_render_queue_buffer_);
902
903 // Insert the samples into the queue.
904 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
905 // The data queue is full and needs to be emptied.
906 EmptyQueuedRenderAudio();
907
908 // Retry the insert (should always work).
909 bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
910 RTC_DCHECK(result);
911 }
912
913 if (!constants_.use_experimental_agc) {
914 GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_);
915 // Insert the samples into the queue.
916 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
917 // The data queue is full and needs to be emptied.
918 EmptyQueuedRenderAudio();
919
920 // Retry the insert (should always work).
921 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
922 RTC_DCHECK(result);
923 }
924 }
925 }
926
QueueNonbandedRenderAudio(AudioBuffer * audio)927 void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
928 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
929
930 // Insert the samples into the queue.
931 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
932 // The data queue is full and needs to be emptied.
933 EmptyQueuedRenderAudio();
934
935 // Retry the insert (should always work).
936 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
937 RTC_DCHECK(result);
938 }
939 }
940
AllocateRenderQueue()941 void AudioProcessingImpl::AllocateRenderQueue() {
942 const size_t new_aec_render_queue_element_max_size =
943 std::max(static_cast<size_t>(1),
944 kMaxAllowedValuesOfSamplesPerBand *
945 EchoCancellationImpl::NumCancellersRequired(
946 num_output_channels(), num_reverse_channels()));
947
948 const size_t new_aecm_render_queue_element_max_size =
949 std::max(static_cast<size_t>(1),
950 kMaxAllowedValuesOfSamplesPerBand *
951 EchoControlMobileImpl::NumCancellersRequired(
952 num_output_channels(), num_reverse_channels()));
953
954 const size_t new_agc_render_queue_element_max_size =
955 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
956
957 const size_t new_red_render_queue_element_max_size =
958 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
959
960 // Reallocate the queues if the queue item sizes are too small to fit the
961 // data to put in the queues.
962 if (aec_render_queue_element_max_size_ <
963 new_aec_render_queue_element_max_size) {
964 aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
965
966 std::vector<float> template_queue_element(
967 aec_render_queue_element_max_size_);
968
969 aec_render_signal_queue_.reset(
970 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
971 kMaxNumFramesToBuffer, template_queue_element,
972 RenderQueueItemVerifier<float>(
973 aec_render_queue_element_max_size_)));
974
975 aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
976 aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
977 } else {
978 aec_render_signal_queue_->Clear();
979 }
980
981 if (aecm_render_queue_element_max_size_ <
982 new_aecm_render_queue_element_max_size) {
983 aecm_render_queue_element_max_size_ =
984 new_aecm_render_queue_element_max_size;
985
986 std::vector<int16_t> template_queue_element(
987 aecm_render_queue_element_max_size_);
988
989 aecm_render_signal_queue_.reset(
990 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
991 kMaxNumFramesToBuffer, template_queue_element,
992 RenderQueueItemVerifier<int16_t>(
993 aecm_render_queue_element_max_size_)));
994
995 aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_);
996 aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_);
997 } else {
998 aecm_render_signal_queue_->Clear();
999 }
1000
1001 if (agc_render_queue_element_max_size_ <
1002 new_agc_render_queue_element_max_size) {
1003 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
1004
1005 std::vector<int16_t> template_queue_element(
1006 agc_render_queue_element_max_size_);
1007
1008 agc_render_signal_queue_.reset(
1009 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1010 kMaxNumFramesToBuffer, template_queue_element,
1011 RenderQueueItemVerifier<int16_t>(
1012 agc_render_queue_element_max_size_)));
1013
1014 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
1015 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
1016 } else {
1017 agc_render_signal_queue_->Clear();
1018 }
1019
1020 if (red_render_queue_element_max_size_ <
1021 new_red_render_queue_element_max_size) {
1022 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
1023
1024 std::vector<float> template_queue_element(
1025 red_render_queue_element_max_size_);
1026
1027 red_render_signal_queue_.reset(
1028 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1029 kMaxNumFramesToBuffer, template_queue_element,
1030 RenderQueueItemVerifier<float>(
1031 red_render_queue_element_max_size_)));
1032
1033 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
1034 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
1035 } else {
1036 red_render_signal_queue_->Clear();
1037 }
1038 }
1039
EmptyQueuedRenderAudio()1040 void AudioProcessingImpl::EmptyQueuedRenderAudio() {
1041 rtc::CritScope cs_capture(&crit_capture_);
1042 while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
1043 public_submodules_->echo_cancellation->ProcessRenderAudio(
1044 aec_capture_queue_buffer_);
1045 }
1046
1047 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
1048 public_submodules_->echo_control_mobile->ProcessRenderAudio(
1049 aecm_capture_queue_buffer_);
1050 }
1051
1052 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1053 public_submodules_->gain_control->ProcessRenderAudio(
1054 agc_capture_queue_buffer_);
1055 }
1056
1057 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
1058 private_submodules_->residual_echo_detector->AnalyzeRenderAudio(
1059 red_capture_queue_buffer_);
1060 }
1061 }
1062
ProcessStream(AudioFrame * frame)1063 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
1064 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
1065 {
1066 // Acquire the capture lock in order to safely call the function
1067 // that retrieves the render side data. This function accesses apm
1068 // getters that need the capture lock held when being called.
1069 // The lock needs to be released as
1070 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock
1071 // as well.
1072 rtc::CritScope cs_capture(&crit_capture_);
1073 EmptyQueuedRenderAudio();
1074 }
1075
1076 if (!frame) {
1077 return kNullPointerError;
1078 }
1079 // Must be a native rate.
1080 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
1081 frame->sample_rate_hz_ != kSampleRate16kHz &&
1082 frame->sample_rate_hz_ != kSampleRate32kHz &&
1083 frame->sample_rate_hz_ != kSampleRate48kHz) {
1084 return kBadSampleRateError;
1085 }
1086
1087 ProcessingConfig processing_config;
1088 bool reinitialization_required = false;
1089 {
1090 // Aquire lock for the access of api_format.
1091 // The lock is released immediately due to the conditional
1092 // reinitialization.
1093 rtc::CritScope cs_capture(&crit_capture_);
1094 // TODO(ajm): The input and output rates and channels are currently
1095 // constrained to be identical in the int16 interface.
1096 processing_config = formats_.api_format;
1097
1098 reinitialization_required = UpdateActiveSubmoduleStates();
1099 }
1100 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_);
1101 processing_config.input_stream().set_num_channels(frame->num_channels_);
1102 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_);
1103 processing_config.output_stream().set_num_channels(frame->num_channels_);
1104
1105 {
1106 // Do conditional reinitialization.
1107 rtc::CritScope cs_render(&crit_render_);
1108 RETURN_ON_ERR(
1109 MaybeInitializeCapture(processing_config, reinitialization_required));
1110 }
1111 rtc::CritScope cs_capture(&crit_capture_);
1112 if (frame->samples_per_channel_ !=
1113 formats_.api_format.input_stream().num_frames()) {
1114 return kBadDataLengthError;
1115 }
1116
1117 if (aec_dump_) {
1118 RecordUnprocessedCaptureStream(*frame);
1119 }
1120
1121 capture_.capture_audio->DeinterleaveFrom(frame);
1122 RETURN_ON_ERR(ProcessCaptureStreamLocked());
1123 capture_.capture_audio->InterleaveTo(
1124 frame, submodule_states_.CaptureMultiBandProcessingActive() ||
1125 submodule_states_.CaptureFullBandProcessingActive());
1126
1127 if (aec_dump_) {
1128 RecordProcessedCaptureStream(*frame);
1129 }
1130
1131 return kNoError;
1132 }
1133
ProcessCaptureStreamLocked()1134 int AudioProcessingImpl::ProcessCaptureStreamLocked() {
1135 // Ensure that not both the AEC and AECM are active at the same time.
1136 // TODO(peah): Simplify once the public API Enable functions for these
1137 // are moved to APM.
1138 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() &&
1139 public_submodules_->echo_control_mobile->is_enabled()));
1140
1141 MaybeUpdateHistograms();
1142
1143 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
1144
1145 capture_input_rms_.Analyze(rtc::ArrayView<const int16_t>(
1146 capture_buffer->channels_const()[0],
1147 capture_nonlocked_.capture_processing_format.num_frames()));
1148 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1149 if (log_rms) {
1150 capture_rms_interval_counter_ = 0;
1151 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
1152 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1153 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1154 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1155 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1156 }
1157
1158 if (private_submodules_->echo_controller) {
1159 // TODO(peah): Reactivate analogue AGC gain detection once the analogue AGC
1160 // issues have been addressed.
1161 capture_.echo_path_gain_change = false;
1162 private_submodules_->echo_controller->AnalyzeCapture(capture_buffer);
1163 }
1164
1165 if (constants_.use_experimental_agc &&
1166 public_submodules_->gain_control->is_enabled()) {
1167 private_submodules_->agc_manager->AnalyzePreProcess(
1168 capture_buffer->channels()[0], capture_buffer->num_channels(),
1169 capture_nonlocked_.capture_processing_format.num_frames());
1170 }
1171
1172 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1173 SampleRateSupportsMultiBand(
1174 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1175 capture_buffer->SplitIntoFrequencyBands();
1176 }
1177
1178 if (private_submodules_->echo_controller) {
1179 // Force down-mixing of the number of channels after the detection of
1180 // capture signal saturation.
1181 // TODO(peah): Look into ensuring that this kind of tampering with the
1182 // AudioBuffer functionality should not be needed.
1183 capture_buffer->set_num_channels(1);
1184 }
1185
1186 if (capture_nonlocked_.beamformer_enabled) {
1187 private_submodules_->beamformer->AnalyzeChunk(
1188 *capture_buffer->split_data_f());
1189 // Discards all channels by the leftmost one.
1190 capture_buffer->set_num_channels(1);
1191 }
1192
1193 // TODO(peah): Move the AEC3 low-cut filter to this place.
1194 if (private_submodules_->low_cut_filter &&
1195 !private_submodules_->echo_controller) {
1196 private_submodules_->low_cut_filter->Process(capture_buffer);
1197 }
1198 RETURN_ON_ERR(
1199 public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer));
1200 public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer);
1201
1202 // Ensure that the stream delay was set before the call to the
1203 // AEC ProcessCaptureAudio function.
1204 if (public_submodules_->echo_cancellation->is_enabled() &&
1205 !was_stream_delay_set()) {
1206 return AudioProcessing::kStreamParameterNotSetError;
1207 }
1208
1209 if (private_submodules_->echo_controller) {
1210 private_submodules_->echo_controller->ProcessCapture(
1211 capture_buffer, capture_.echo_path_gain_change);
1212 } else {
1213 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(
1214 capture_buffer, stream_delay_ms()));
1215 }
1216
1217 if (public_submodules_->echo_control_mobile->is_enabled() &&
1218 public_submodules_->noise_suppression->is_enabled()) {
1219 capture_buffer->CopyLowPassToReference();
1220 }
1221 public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
1222 #if WEBRTC_INTELLIGIBILITY_ENHANCER
1223 if (capture_nonlocked_.intelligibility_enabled) {
1224 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled());
1225 int gain_db = public_submodules_->gain_control->is_enabled() ?
1226 public_submodules_->gain_control->compression_gain_db() :
1227 0;
1228 float gain = std::pow(10.f, gain_db / 20.f);
1229 gain *= capture_nonlocked_.level_controller_enabled ?
1230 private_submodules_->level_controller->GetLastGain() :
1231 1.f;
1232 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate(
1233 public_submodules_->noise_suppression->NoiseEstimate(), gain);
1234 }
1235 #endif
1236
1237 // Ensure that the stream delay was set before the call to the
1238 // AECM ProcessCaptureAudio function.
1239 if (public_submodules_->echo_control_mobile->is_enabled() &&
1240 !was_stream_delay_set()) {
1241 return AudioProcessing::kStreamParameterNotSetError;
1242 }
1243
1244 if (!(private_submodules_->echo_controller ||
1245 public_submodules_->echo_cancellation->is_enabled())) {
1246 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio(
1247 capture_buffer, stream_delay_ms()));
1248 }
1249
1250 if (capture_nonlocked_.beamformer_enabled) {
1251 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f());
1252 }
1253
1254 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer);
1255
1256 if (constants_.use_experimental_agc &&
1257 public_submodules_->gain_control->is_enabled() &&
1258 (!capture_nonlocked_.beamformer_enabled ||
1259 private_submodules_->beamformer->is_target_present())) {
1260 private_submodules_->agc_manager->Process(
1261 capture_buffer->split_bands_const(0)[kBand0To8kHz],
1262 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate);
1263 }
1264 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(
1265 capture_buffer, echo_cancellation()->stream_has_echo()));
1266
1267 if (submodule_states_.CaptureMultiBandProcessingActive() &&
1268 SampleRateSupportsMultiBand(
1269 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1270 capture_buffer->MergeFrequencyBands();
1271 }
1272
1273 if (config_.residual_echo_detector.enabled) {
1274 private_submodules_->residual_echo_detector->AnalyzeCaptureAudio(
1275 rtc::ArrayView<const float>(capture_buffer->channels_f()[0],
1276 capture_buffer->num_frames()));
1277 }
1278
1279 // TODO(aluebs): Investigate if the transient suppression placement should be
1280 // before or after the AGC.
1281 if (capture_.transient_suppressor_enabled) {
1282 float voice_probability =
1283 private_submodules_->agc_manager.get()
1284 ? private_submodules_->agc_manager->voice_probability()
1285 : 1.f;
1286
1287 public_submodules_->transient_suppressor->Suppress(
1288 capture_buffer->channels_f()[0], capture_buffer->num_frames(),
1289 capture_buffer->num_channels(),
1290 capture_buffer->split_bands_const_f(0)[kBand0To8kHz],
1291 capture_buffer->num_frames_per_band(), capture_buffer->keyboard_data(),
1292 capture_buffer->num_keyboard_frames(), voice_probability,
1293 capture_.key_pressed);
1294 }
1295
1296 if (config_.gain_controller2.enabled) {
1297 private_submodules_->gain_controller2->Process(capture_buffer);
1298 }
1299
1300 if (capture_nonlocked_.level_controller_enabled) {
1301 private_submodules_->level_controller->Process(capture_buffer);
1302 }
1303
1304 if (private_submodules_->capture_post_processor) {
1305 private_submodules_->capture_post_processor->Process(capture_buffer);
1306 }
1307
1308 // The level estimator operates on the recombined data.
1309 public_submodules_->level_estimator->ProcessStream(capture_buffer);
1310
1311 capture_output_rms_.Analyze(rtc::ArrayView<const int16_t>(
1312 capture_buffer->channels_const()[0],
1313 capture_nonlocked_.capture_processing_format.num_frames()));
1314 if (log_rms) {
1315 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1316 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms",
1317 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1318 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1319 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1320 }
1321
1322 capture_.was_stream_delay_set = false;
1323 return kNoError;
1324 }
1325
AnalyzeReverseStream(const float * const * data,size_t samples_per_channel,int sample_rate_hz,ChannelLayout layout)1326 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
1327 size_t samples_per_channel,
1328 int sample_rate_hz,
1329 ChannelLayout layout) {
1330 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout");
1331 rtc::CritScope cs(&crit_render_);
1332 const StreamConfig reverse_config = {
1333 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout),
1334 };
1335 if (samples_per_channel != reverse_config.num_frames()) {
1336 return kBadDataLengthError;
1337 }
1338 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1339 }
1340
ProcessReverseStream(const float * const * src,const StreamConfig & input_config,const StreamConfig & output_config,float * const * dest)1341 int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1342 const StreamConfig& input_config,
1343 const StreamConfig& output_config,
1344 float* const* dest) {
1345 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
1346 rtc::CritScope cs(&crit_render_);
1347 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
1348 if (submodule_states_.RenderMultiBandProcessingActive()) {
1349 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1350 dest);
1351 } else if (formats_.api_format.reverse_input_stream() !=
1352 formats_.api_format.reverse_output_stream()) {
1353 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1354 output_config.num_samples());
1355 } else {
1356 CopyAudioIfNeeded(src, input_config.num_frames(),
1357 input_config.num_channels(), dest);
1358 }
1359
1360 return kNoError;
1361 }
1362
AnalyzeReverseStreamLocked(const float * const * src,const StreamConfig & input_config,const StreamConfig & output_config)1363 int AudioProcessingImpl::AnalyzeReverseStreamLocked(
1364 const float* const* src,
1365 const StreamConfig& input_config,
1366 const StreamConfig& output_config) {
1367 if (src == nullptr) {
1368 return kNullPointerError;
1369 }
1370
1371 if (input_config.num_channels() == 0) {
1372 return kBadNumberChannelsError;
1373 }
1374
1375 ProcessingConfig processing_config = formats_.api_format;
1376 processing_config.reverse_input_stream() = input_config;
1377 processing_config.reverse_output_stream() = output_config;
1378
1379 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
1380 assert(input_config.num_frames() ==
1381 formats_.api_format.reverse_input_stream().num_frames());
1382
1383 if (aec_dump_) {
1384 const size_t channel_size =
1385 formats_.api_format.reverse_input_stream().num_frames();
1386 const size_t num_channels =
1387 formats_.api_format.reverse_input_stream().num_channels();
1388 aec_dump_->WriteRenderStreamMessage(
1389 FloatAudioFrame(src, num_channels, channel_size));
1390 }
1391 render_.render_audio->CopyFrom(src,
1392 formats_.api_format.reverse_input_stream());
1393 return ProcessRenderStreamLocked();
1394 }
1395
ProcessReverseStream(AudioFrame * frame)1396 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
1397 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
1398 rtc::CritScope cs(&crit_render_);
1399 if (frame == nullptr) {
1400 return kNullPointerError;
1401 }
1402 // Must be a native rate.
1403 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
1404 frame->sample_rate_hz_ != kSampleRate16kHz &&
1405 frame->sample_rate_hz_ != kSampleRate32kHz &&
1406 frame->sample_rate_hz_ != kSampleRate48kHz) {
1407 return kBadSampleRateError;
1408 }
1409
1410 if (frame->num_channels_ <= 0) {
1411 return kBadNumberChannelsError;
1412 }
1413
1414 ProcessingConfig processing_config = formats_.api_format;
1415 processing_config.reverse_input_stream().set_sample_rate_hz(
1416 frame->sample_rate_hz_);
1417 processing_config.reverse_input_stream().set_num_channels(
1418 frame->num_channels_);
1419 processing_config.reverse_output_stream().set_sample_rate_hz(
1420 frame->sample_rate_hz_);
1421 processing_config.reverse_output_stream().set_num_channels(
1422 frame->num_channels_);
1423
1424 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
1425 if (frame->samples_per_channel_ !=
1426 formats_.api_format.reverse_input_stream().num_frames()) {
1427 return kBadDataLengthError;
1428 }
1429
1430 if (aec_dump_) {
1431 aec_dump_->WriteRenderStreamMessage(*frame);
1432 }
1433
1434 render_.render_audio->DeinterleaveFrom(frame);
1435 RETURN_ON_ERR(ProcessRenderStreamLocked());
1436 render_.render_audio->InterleaveTo(
1437 frame, submodule_states_.RenderMultiBandProcessingActive());
1438 return kNoError;
1439 }
1440
ProcessRenderStreamLocked()1441 int AudioProcessingImpl::ProcessRenderStreamLocked() {
1442 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
1443
1444 QueueNonbandedRenderAudio(render_buffer);
1445
1446 if (submodule_states_.RenderMultiBandSubModulesActive() &&
1447 SampleRateSupportsMultiBand(
1448 formats_.render_processing_format.sample_rate_hz())) {
1449 render_buffer->SplitIntoFrequencyBands();
1450 }
1451
1452 #if WEBRTC_INTELLIGIBILITY_ENHANCER
1453 if (capture_nonlocked_.intelligibility_enabled) {
1454 public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
1455 render_buffer);
1456 }
1457 #endif
1458
1459 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1460 QueueBandedRenderAudio(render_buffer);
1461 }
1462
1463 // TODO(peah): Perform the queueing ínside QueueRenderAudiuo().
1464 if (private_submodules_->echo_controller) {
1465 private_submodules_->echo_controller->AnalyzeRender(render_buffer);
1466 }
1467
1468 if (submodule_states_.RenderMultiBandProcessingActive() &&
1469 SampleRateSupportsMultiBand(
1470 formats_.render_processing_format.sample_rate_hz())) {
1471 render_buffer->MergeFrequencyBands();
1472 }
1473
1474 return kNoError;
1475 }
1476
set_stream_delay_ms(int delay)1477 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
1478 rtc::CritScope cs(&crit_capture_);
1479 Error retval = kNoError;
1480 capture_.was_stream_delay_set = true;
1481 delay += capture_.delay_offset_ms;
1482
1483 if (delay < 0) {
1484 delay = 0;
1485 retval = kBadStreamParameterWarning;
1486 }
1487
1488 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1489 if (delay > 500) {
1490 delay = 500;
1491 retval = kBadStreamParameterWarning;
1492 }
1493
1494 capture_nonlocked_.stream_delay_ms = delay;
1495 return retval;
1496 }
1497
stream_delay_ms() const1498 int AudioProcessingImpl::stream_delay_ms() const {
1499 // Used as callback from submodules, hence locking is not allowed.
1500 return capture_nonlocked_.stream_delay_ms;
1501 }
1502
was_stream_delay_set() const1503 bool AudioProcessingImpl::was_stream_delay_set() const {
1504 // Used as callback from submodules, hence locking is not allowed.
1505 return capture_.was_stream_delay_set;
1506 }
1507
set_stream_key_pressed(bool key_pressed)1508 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
1509 rtc::CritScope cs(&crit_capture_);
1510 capture_.key_pressed = key_pressed;
1511 }
1512
set_delay_offset_ms(int offset)1513 void AudioProcessingImpl::set_delay_offset_ms(int offset) {
1514 rtc::CritScope cs(&crit_capture_);
1515 capture_.delay_offset_ms = offset;
1516 }
1517
delay_offset_ms() const1518 int AudioProcessingImpl::delay_offset_ms() const {
1519 rtc::CritScope cs(&crit_capture_);
1520 return capture_.delay_offset_ms;
1521 }
1522
AttachAecDump(std::unique_ptr<AecDump> aec_dump)1523 void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1524 RTC_DCHECK(aec_dump);
1525 rtc::CritScope cs_render(&crit_render_);
1526 rtc::CritScope cs_capture(&crit_capture_);
1527
1528 // The previously attached AecDump will be destroyed with the
1529 // 'aec_dump' parameter, which is after locks are released.
1530 aec_dump_.swap(aec_dump);
1531 WriteAecDumpConfigMessage(true);
1532 aec_dump_->WriteInitMessage(ToStreamsConfig(formats_.api_format));
1533 }
1534
DetachAecDump()1535 void AudioProcessingImpl::DetachAecDump() {
1536 // The d-tor of a task-queue based AecDump blocks until all pending
1537 // tasks are done. This construction avoids blocking while holding
1538 // the render and capture locks.
1539 std::unique_ptr<AecDump> aec_dump = nullptr;
1540 {
1541 rtc::CritScope cs_render(&crit_render_);
1542 rtc::CritScope cs_capture(&crit_capture_);
1543 aec_dump = std::move(aec_dump_);
1544 }
1545 }
1546
AudioProcessingStatistics()1547 AudioProcessing::AudioProcessingStatistics::AudioProcessingStatistics() {
1548 residual_echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f);
1549 echo_return_loss.Set(-100.0f, -100.0f, -100.0f, -100.0f);
1550 echo_return_loss_enhancement.Set(-100.0f, -100.0f, -100.0f, -100.0f);
1551 a_nlp.Set(-100.0f, -100.0f, -100.0f, -100.0f);
1552 }
1553
1554 AudioProcessing::AudioProcessingStatistics::AudioProcessingStatistics(
1555 const AudioProcessingStatistics& other) = default;
1556
1557 AudioProcessing::AudioProcessingStatistics::~AudioProcessingStatistics() =
1558 default;
1559
1560 // TODO(ivoc): Remove this when GetStatistics() becomes pure virtual.
GetStatistics() const1561 AudioProcessing::AudioProcessingStatistics AudioProcessing::GetStatistics()
1562 const {
1563 return AudioProcessingStatistics();
1564 }
1565
1566 // TODO(ivoc): Remove this when GetStatistics() becomes pure virtual.
GetStatistics(bool has_remote_tracks) const1567 AudioProcessingStats AudioProcessing::GetStatistics(
1568 bool has_remote_tracks) const {
1569 return AudioProcessingStats();
1570 }
1571
GetStatistics() const1572 AudioProcessing::AudioProcessingStatistics AudioProcessingImpl::GetStatistics()
1573 const {
1574 AudioProcessingStatistics stats;
1575 EchoCancellation::Metrics metrics;
1576 if (private_submodules_->echo_controller) {
1577 rtc::CritScope cs_capture(&crit_capture_);
1578 auto ec_metrics = private_submodules_->echo_controller->GetMetrics();
1579 float erl = static_cast<float>(ec_metrics.echo_return_loss);
1580 float erle = static_cast<float>(ec_metrics.echo_return_loss_enhancement);
1581 // Instant value will also be used for min, max and average.
1582 stats.echo_return_loss.Set(erl, erl, erl, erl);
1583 stats.echo_return_loss_enhancement.Set(erle, erle, erle, erle);
1584 } else if (public_submodules_->echo_cancellation->GetMetrics(&metrics) ==
1585 Error::kNoError) {
1586 stats.a_nlp.Set(metrics.a_nlp);
1587 stats.divergent_filter_fraction = metrics.divergent_filter_fraction;
1588 stats.echo_return_loss.Set(metrics.echo_return_loss);
1589 stats.echo_return_loss_enhancement.Set(
1590 metrics.echo_return_loss_enhancement);
1591 stats.residual_echo_return_loss.Set(metrics.residual_echo_return_loss);
1592 }
1593 {
1594 rtc::CritScope cs_capture(&crit_capture_);
1595 stats.residual_echo_likelihood =
1596 private_submodules_->residual_echo_detector->echo_likelihood();
1597 stats.residual_echo_likelihood_recent_max =
1598 private_submodules_->residual_echo_detector
1599 ->echo_likelihood_recent_max();
1600 }
1601 public_submodules_->echo_cancellation->GetDelayMetrics(
1602 &stats.delay_median, &stats.delay_standard_deviation,
1603 &stats.fraction_poor_delays);
1604 return stats;
1605 }
1606
GetStatistics(bool has_remote_tracks) const1607 AudioProcessingStats AudioProcessingImpl::GetStatistics(
1608 bool has_remote_tracks) const {
1609 AudioProcessingStats stats;
1610 if (has_remote_tracks) {
1611 EchoCancellation::Metrics metrics;
1612 if (private_submodules_->echo_controller) {
1613 rtc::CritScope cs_capture(&crit_capture_);
1614 auto ec_metrics = private_submodules_->echo_controller->GetMetrics();
1615 stats.echo_return_loss = ec_metrics.echo_return_loss;
1616 stats.echo_return_loss_enhancement =
1617 ec_metrics.echo_return_loss_enhancement;
1618 stats.delay_ms = ec_metrics.delay_ms;
1619 } else if (public_submodules_->echo_cancellation->GetMetrics(&metrics) ==
1620 Error::kNoError) {
1621 if (metrics.divergent_filter_fraction != -1.0f) {
1622 stats.divergent_filter_fraction =
1623 rtc::Optional<double>(metrics.divergent_filter_fraction);
1624 }
1625 if (metrics.echo_return_loss.instant != -100) {
1626 stats.echo_return_loss =
1627 rtc::Optional<double>(metrics.echo_return_loss.instant);
1628 }
1629 if (metrics.echo_return_loss_enhancement.instant != -100) {
1630 stats.echo_return_loss_enhancement =
1631 rtc::Optional<double>(metrics.echo_return_loss_enhancement.instant);
1632 }
1633 }
1634 if (config_.residual_echo_detector.enabled) {
1635 rtc::CritScope cs_capture(&crit_capture_);
1636 stats.residual_echo_likelihood = rtc::Optional<double>(
1637 private_submodules_->residual_echo_detector->echo_likelihood());
1638 stats.residual_echo_likelihood_recent_max =
1639 rtc::Optional<double>(private_submodules_->residual_echo_detector
1640 ->echo_likelihood_recent_max());
1641 }
1642 int delay_median, delay_std;
1643 float fraction_poor_delays;
1644 if (public_submodules_->echo_cancellation->GetDelayMetrics(
1645 &delay_median, &delay_std, &fraction_poor_delays) ==
1646 Error::kNoError) {
1647 if (delay_median >= 0) {
1648 stats.delay_median_ms = rtc::Optional<int32_t>(delay_median);
1649 }
1650 if (delay_std >= 0) {
1651 stats.delay_standard_deviation_ms = rtc::Optional<int32_t>(delay_std);
1652 }
1653 }
1654 }
1655 return stats;
1656 }
1657
echo_cancellation() const1658 EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
1659 return public_submodules_->echo_cancellation.get();
1660 }
1661
echo_control_mobile() const1662 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
1663 return public_submodules_->echo_control_mobile.get();
1664 }
1665
gain_control() const1666 GainControl* AudioProcessingImpl::gain_control() const {
1667 if (constants_.use_experimental_agc) {
1668 return public_submodules_->gain_control_for_experimental_agc.get();
1669 }
1670 return public_submodules_->gain_control.get();
1671 }
1672
high_pass_filter() const1673 HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
1674 return high_pass_filter_impl_.get();
1675 }
1676
level_estimator() const1677 LevelEstimator* AudioProcessingImpl::level_estimator() const {
1678 return public_submodules_->level_estimator.get();
1679 }
1680
noise_suppression() const1681 NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
1682 return public_submodules_->noise_suppression.get();
1683 }
1684
voice_detection() const1685 VoiceDetection* AudioProcessingImpl::voice_detection() const {
1686 return public_submodules_->voice_detection.get();
1687 }
1688
MutateConfig(rtc::FunctionView<void (AudioProcessing::Config *)> mutator)1689 void AudioProcessingImpl::MutateConfig(
1690 rtc::FunctionView<void(AudioProcessing::Config*)> mutator) {
1691 rtc::CritScope cs_render(&crit_render_);
1692 rtc::CritScope cs_capture(&crit_capture_);
1693 mutator(&config_);
1694 ApplyConfig(config_);
1695 }
1696
GetConfig() const1697 AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
1698 rtc::CritScope cs_render(&crit_render_);
1699 rtc::CritScope cs_capture(&crit_capture_);
1700 return config_;
1701 }
1702
UpdateActiveSubmoduleStates()1703 bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1704 return submodule_states_.Update(
1705 config_.high_pass_filter.enabled,
1706 public_submodules_->echo_cancellation->is_enabled(),
1707 public_submodules_->echo_control_mobile->is_enabled(),
1708 config_.residual_echo_detector.enabled,
1709 public_submodules_->noise_suppression->is_enabled(),
1710 capture_nonlocked_.intelligibility_enabled,
1711 capture_nonlocked_.beamformer_enabled,
1712 public_submodules_->gain_control->is_enabled(),
1713 config_.gain_controller2.enabled,
1714 capture_nonlocked_.level_controller_enabled,
1715 capture_nonlocked_.echo_controller_enabled,
1716 public_submodules_->voice_detection->is_enabled(),
1717 public_submodules_->level_estimator->is_enabled(),
1718 capture_.transient_suppressor_enabled);
1719 }
1720
1721
InitializeTransient()1722 void AudioProcessingImpl::InitializeTransient() {
1723 if (capture_.transient_suppressor_enabled) {
1724 if (!public_submodules_->transient_suppressor.get()) {
1725 public_submodules_->transient_suppressor.reset(new TransientSuppressor());
1726 }
1727 public_submodules_->transient_suppressor->Initialize(
1728 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
1729 capture_nonlocked_.split_rate, num_proc_channels());
1730 }
1731 }
1732
InitializeBeamformer()1733 void AudioProcessingImpl::InitializeBeamformer() {
1734 if (capture_nonlocked_.beamformer_enabled) {
1735 if (!private_submodules_->beamformer) {
1736 private_submodules_->beamformer.reset(new NonlinearBeamformer(
1737 capture_.array_geometry, 1u, capture_.target_direction));
1738 }
1739 private_submodules_->beamformer->Initialize(kChunkSizeMs,
1740 capture_nonlocked_.split_rate);
1741 }
1742 }
1743
InitializeIntelligibility()1744 void AudioProcessingImpl::InitializeIntelligibility() {
1745 #if WEBRTC_INTELLIGIBILITY_ENHANCER
1746 if (capture_nonlocked_.intelligibility_enabled) {
1747 public_submodules_->intelligibility_enhancer.reset(
1748 new IntelligibilityEnhancer(capture_nonlocked_.split_rate,
1749 render_.render_audio->num_channels(),
1750 render_.render_audio->num_bands(),
1751 NoiseSuppressionImpl::num_noise_bins()));
1752 }
1753 #endif
1754 }
1755
InitializeLowCutFilter()1756 void AudioProcessingImpl::InitializeLowCutFilter() {
1757 if (config_.high_pass_filter.enabled) {
1758 private_submodules_->low_cut_filter.reset(
1759 new LowCutFilter(num_proc_channels(), proc_sample_rate_hz()));
1760 } else {
1761 private_submodules_->low_cut_filter.reset();
1762 }
1763 }
1764
InitializeEchoController()1765 void AudioProcessingImpl::InitializeEchoController() {
1766 if (echo_control_factory_) {
1767 private_submodules_->echo_controller =
1768 echo_control_factory_->Create(proc_sample_rate_hz());
1769 } else {
1770 private_submodules_->echo_controller.reset();
1771 }
1772 }
1773
InitializeGainController2()1774 void AudioProcessingImpl::InitializeGainController2() {
1775 if (config_.gain_controller2.enabled) {
1776 private_submodules_->gain_controller2->Initialize(proc_sample_rate_hz());
1777 }
1778 }
1779
InitializeLevelController()1780 void AudioProcessingImpl::InitializeLevelController() {
1781 private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
1782 }
1783
InitializeResidualEchoDetector()1784 void AudioProcessingImpl::InitializeResidualEchoDetector() {
1785 private_submodules_->residual_echo_detector->Initialize();
1786 }
1787
InitializePostProcessor()1788 void AudioProcessingImpl::InitializePostProcessor() {
1789 if (private_submodules_->capture_post_processor) {
1790 private_submodules_->capture_post_processor->Initialize(
1791 proc_sample_rate_hz(), num_proc_channels());
1792 }
1793 }
1794
MaybeUpdateHistograms()1795 void AudioProcessingImpl::MaybeUpdateHistograms() {
1796 static const int kMinDiffDelayMs = 60;
1797
1798 if (echo_cancellation()->is_enabled()) {
1799 // Activate delay_jumps_ counters if we know echo_cancellation is running.
1800 // If a stream has echo we know that the echo_cancellation is in process.
1801 if (capture_.stream_delay_jumps == -1 &&
1802 echo_cancellation()->stream_has_echo()) {
1803 capture_.stream_delay_jumps = 0;
1804 }
1805 if (capture_.aec_system_delay_jumps == -1 &&
1806 echo_cancellation()->stream_has_echo()) {
1807 capture_.aec_system_delay_jumps = 0;
1808 }
1809
1810 // Detect a jump in platform reported system delay and log the difference.
1811 const int diff_stream_delay_ms =
1812 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms;
1813 if (diff_stream_delay_ms > kMinDiffDelayMs &&
1814 capture_.last_stream_delay_ms != 0) {
1815 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump",
1816 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100);
1817 if (capture_.stream_delay_jumps == -1) {
1818 capture_.stream_delay_jumps = 0; // Activate counter if needed.
1819 }
1820 capture_.stream_delay_jumps++;
1821 }
1822 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms;
1823
1824 // Detect a jump in AEC system delay and log the difference.
1825 const int samples_per_ms =
1826 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000);
1827 RTC_DCHECK_LT(0, samples_per_ms);
1828 const int aec_system_delay_ms =
1829 public_submodules_->echo_cancellation->GetSystemDelayInSamples() /
1830 samples_per_ms;
1831 const int diff_aec_system_delay_ms =
1832 aec_system_delay_ms - capture_.last_aec_system_delay_ms;
1833 if (diff_aec_system_delay_ms > kMinDiffDelayMs &&
1834 capture_.last_aec_system_delay_ms != 0) {
1835 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump",
1836 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000,
1837 100);
1838 if (capture_.aec_system_delay_jumps == -1) {
1839 capture_.aec_system_delay_jumps = 0; // Activate counter if needed.
1840 }
1841 capture_.aec_system_delay_jumps++;
1842 }
1843 capture_.last_aec_system_delay_ms = aec_system_delay_ms;
1844 }
1845 }
1846
UpdateHistogramsOnCallEnd()1847 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() {
1848 // Run in a single-threaded manner.
1849 rtc::CritScope cs_render(&crit_render_);
1850 rtc::CritScope cs_capture(&crit_capture_);
1851
1852 if (capture_.stream_delay_jumps > -1) {
1853 RTC_HISTOGRAM_ENUMERATION(
1854 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps",
1855 capture_.stream_delay_jumps, 51);
1856 }
1857 capture_.stream_delay_jumps = -1;
1858 capture_.last_stream_delay_ms = 0;
1859
1860 if (capture_.aec_system_delay_jumps > -1) {
1861 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps",
1862 capture_.aec_system_delay_jumps, 51);
1863 }
1864 capture_.aec_system_delay_jumps = -1;
1865 capture_.last_aec_system_delay_ms = 0;
1866 }
1867
WriteAecDumpConfigMessage(bool forced)1868 void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
1869 if (!aec_dump_) {
1870 return;
1871 }
1872 std::string experiments_description =
1873 public_submodules_->echo_cancellation->GetExperimentsDescription();
1874 // TODO(peah): Add semicolon-separated concatenations of experiment
1875 // descriptions for other submodules.
1876 if (capture_nonlocked_.level_controller_enabled) {
1877 experiments_description += "LevelController;";
1878 }
1879 if (constants_.agc_clipped_level_min != kClippedLevelMin) {
1880 experiments_description += "AgcClippingLevelExperiment;";
1881 }
1882 if (capture_nonlocked_.echo_controller_enabled) {
1883 experiments_description += "EchoController;";
1884 }
1885 if (config_.gain_controller2.enabled) {
1886 experiments_description += "GainController2;";
1887 }
1888
1889 InternalAPMConfig apm_config;
1890
1891 apm_config.aec_enabled = public_submodules_->echo_cancellation->is_enabled();
1892 apm_config.aec_delay_agnostic_enabled =
1893 public_submodules_->echo_cancellation->is_delay_agnostic_enabled();
1894 apm_config.aec_drift_compensation_enabled =
1895 public_submodules_->echo_cancellation->is_drift_compensation_enabled();
1896 apm_config.aec_extended_filter_enabled =
1897 public_submodules_->echo_cancellation->is_extended_filter_enabled();
1898 apm_config.aec_suppression_level = static_cast<int>(
1899 public_submodules_->echo_cancellation->suppression_level());
1900
1901 apm_config.aecm_enabled =
1902 public_submodules_->echo_control_mobile->is_enabled();
1903 apm_config.aecm_comfort_noise_enabled =
1904 public_submodules_->echo_control_mobile->is_comfort_noise_enabled();
1905 apm_config.aecm_routing_mode =
1906 static_cast<int>(public_submodules_->echo_control_mobile->routing_mode());
1907
1908 apm_config.agc_enabled = public_submodules_->gain_control->is_enabled();
1909 apm_config.agc_mode =
1910 static_cast<int>(public_submodules_->gain_control->mode());
1911 apm_config.agc_limiter_enabled =
1912 public_submodules_->gain_control->is_limiter_enabled();
1913 apm_config.noise_robust_agc_enabled = constants_.use_experimental_agc;
1914
1915 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
1916
1917 apm_config.ns_enabled = public_submodules_->noise_suppression->is_enabled();
1918 apm_config.ns_level =
1919 static_cast<int>(public_submodules_->noise_suppression->level());
1920
1921 apm_config.transient_suppression_enabled =
1922 capture_.transient_suppressor_enabled;
1923 apm_config.intelligibility_enhancer_enabled =
1924 capture_nonlocked_.intelligibility_enabled;
1925 apm_config.experiments_description = experiments_description;
1926
1927 if (!forced && apm_config == apm_config_for_aec_dump_) {
1928 return;
1929 }
1930 aec_dump_->WriteConfig(apm_config);
1931 apm_config_for_aec_dump_ = apm_config;
1932 }
1933
RecordUnprocessedCaptureStream(const float * const * src)1934 void AudioProcessingImpl::RecordUnprocessedCaptureStream(
1935 const float* const* src) {
1936 RTC_DCHECK(aec_dump_);
1937 WriteAecDumpConfigMessage(false);
1938
1939 const size_t channel_size = formats_.api_format.input_stream().num_frames();
1940 const size_t num_channels = formats_.api_format.input_stream().num_channels();
1941 aec_dump_->AddCaptureStreamInput(
1942 FloatAudioFrame(src, num_channels, channel_size));
1943 RecordAudioProcessingState();
1944 }
1945
RecordUnprocessedCaptureStream(const AudioFrame & capture_frame)1946 void AudioProcessingImpl::RecordUnprocessedCaptureStream(
1947 const AudioFrame& capture_frame) {
1948 RTC_DCHECK(aec_dump_);
1949 WriteAecDumpConfigMessage(false);
1950
1951 aec_dump_->AddCaptureStreamInput(capture_frame);
1952 RecordAudioProcessingState();
1953 }
1954
RecordProcessedCaptureStream(const float * const * processed_capture_stream)1955 void AudioProcessingImpl::RecordProcessedCaptureStream(
1956 const float* const* processed_capture_stream) {
1957 RTC_DCHECK(aec_dump_);
1958
1959 const size_t channel_size = formats_.api_format.output_stream().num_frames();
1960 const size_t num_channels =
1961 formats_.api_format.output_stream().num_channels();
1962 aec_dump_->AddCaptureStreamOutput(
1963 FloatAudioFrame(processed_capture_stream, num_channels, channel_size));
1964 aec_dump_->WriteCaptureStreamMessage();
1965 }
1966
RecordProcessedCaptureStream(const AudioFrame & processed_capture_frame)1967 void AudioProcessingImpl::RecordProcessedCaptureStream(
1968 const AudioFrame& processed_capture_frame) {
1969 RTC_DCHECK(aec_dump_);
1970
1971 aec_dump_->AddCaptureStreamOutput(processed_capture_frame);
1972 aec_dump_->WriteCaptureStreamMessage();
1973 }
1974
RecordAudioProcessingState()1975 void AudioProcessingImpl::RecordAudioProcessingState() {
1976 RTC_DCHECK(aec_dump_);
1977 AecDump::AudioProcessingState audio_proc_state;
1978 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
1979 audio_proc_state.drift =
1980 public_submodules_->echo_cancellation->stream_drift_samples();
1981 audio_proc_state.level = gain_control()->stream_analog_level();
1982 audio_proc_state.keypress = capture_.key_pressed;
1983 aec_dump_->AddAudioProcessingState(audio_proc_state);
1984 }
1985
ApmCaptureState(bool transient_suppressor_enabled,const std::vector<Point> & array_geometry,SphericalPointf target_direction)1986 AudioProcessingImpl::ApmCaptureState::ApmCaptureState(
1987 bool transient_suppressor_enabled,
1988 const std::vector<Point>& array_geometry,
1989 SphericalPointf target_direction)
1990 : aec_system_delay_jumps(-1),
1991 delay_offset_ms(0),
1992 was_stream_delay_set(false),
1993 last_stream_delay_ms(0),
1994 last_aec_system_delay_ms(0),
1995 stream_delay_jumps(-1),
1996 output_will_be_muted(false),
1997 key_pressed(false),
1998 transient_suppressor_enabled(transient_suppressor_enabled),
1999 array_geometry(array_geometry),
2000 target_direction(target_direction),
2001 capture_processing_format(kSampleRate16kHz),
2002 split_rate(kSampleRate16kHz),
2003 echo_path_gain_change(false) {}
2004
2005 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2006
2007 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2008
2009 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2010
2011 } // namespace webrtc
2012